diff --git a/gni/angle.gni b/gni/angle.gni index 92d2fa328f5d2..856fc2b118abc 100644 --- a/gni/angle.gni +++ b/gni/angle.gni @@ -189,11 +189,14 @@ set_defaults("angle_test") { public_deps = [] sources = [] data = [] + defines = [] main = "" suppressed_configs = angle_remove_configs - # TODO(jmadill): Migrate to standalone harness. http://anglebug.com/3162 - if (build_with_chromium) { + # By default use the Chromium harness in Chromium. Can be overriden in a target. + standalone_harness = !build_with_chromium + + if (!standalone_harness) { suppressed_configs -= [ "//build/config/compiler:default_include_dirs" ] } @@ -286,18 +289,6 @@ template("angle_static_library") { } template("angle_test") { - _googletest_deps = [ - "//testing/gmock", - "//testing/gtest", - "//third_party/googletest:gmock", - "//third_party/googletest:gtest", - ] - - # TODO(jmadill): Migrate to standalone harness. http://anglebug.com/3162 - if (build_with_chromium) { - _googletest_deps += [ "//base/test:test_support" ] - } - test(target_name) { forward_variables_from(invoker, "*", @@ -311,9 +302,10 @@ template("angle_test") { forward_variables_from(invoker, [ "visibility" ]) configs += invoker.configs - configs -= invoker.suppressed_configs - configs -= [ angle_root + ":constructor_and_destructor_warnings" ] - configs -= [ angle_root + ":extra_warnings" ] + configs -= invoker.suppressed_configs + [ + "$angle_root:constructor_and_destructor_warnings", + "$angle_root:extra_warnings", + ] if (is_linux && !is_component_build) { # Set rpath to find shared libs in a non-component build. @@ -321,22 +313,40 @@ template("angle_test") { } if (is_android) { - configs += [ angle_root + ":build_id_config" ] - if (build_with_chromium) { - configs -= [ "//build/config/android:hide_all_but_jni" ] - } + configs += [ "$angle_root:build_id_config" ] } - deps += _googletest_deps + [ - "$angle_root:angle_common", - "$angle_root:includes", - "$angle_root/util:angle_test_utils", - ] - - if (build_with_chromium) { - sources += [ "//gpu/${invoker.main}.cc" ] + deps += [ + "$angle_root:angle_common", + "$angle_root:includes", + "$angle_root/third_party/rapidjson:rapidjson", + "$angle_root/util:angle_test_utils", + "//testing/gmock", + "//testing/gtest", + "//third_party/googletest:gmock", + "//third_party/googletest:gtest", + ] + + sources += [ + "$angle_root/src/tests/test_utils/runner/TestSuite.cpp", + "$angle_root/src/tests/test_utils/runner/TestSuite.h", + ] + + # To use the Chromium test infrastructure we must currently use the //base test launcher. + # Eventually we could switch to using standalone testing. See http://crbug.com/837741 + if (standalone_harness) { + if (invoker.main != "") { + sources += [ "${invoker.main}.cpp" ] + } } else { - sources += [ "${invoker.main}.cpp" ] + if (invoker.main != "") { + sources += [ "//gpu/${invoker.main}.cc" ] + } + deps += [ "//base/test:test_support" ] + + if (is_android) { + configs -= [ "//build/config/android:hide_all_but_jni" ] + } } } } diff --git a/src/common/platform.h b/src/common/platform.h index 60db96d0b590f..b82c50620c697 100644 --- a/src/common/platform.h +++ b/src/common/platform.h @@ -123,4 +123,11 @@ # endif #endif +// Define ANGLE_WITH_ASAN macro. +#if defined(__has_feature) +# if __has_feature(address_sanitizer) +# define ANGLE_WITH_ASAN 1 +# endif +#endif + #endif // COMMON_PLATFORM_H_ diff --git a/src/common/system_utils.h b/src/common/system_utils.h index ccb4e9df8d5cf..c0607ac012c92 100644 --- a/src/common/system_utils.h +++ b/src/common/system_utils.h @@ -17,6 +17,8 @@ namespace angle std::string GetExecutablePath(); std::string GetExecutableDirectory(); const char *GetSharedLibraryExtension(); +const char *GetExecutableExtension(); +char GetPathSeparator(); Optional GetCWD(); bool SetCWD(const char *dirName); bool SetEnvironmentVar(const char *variableName, const char *value); diff --git a/src/common/system_utils_posix.cpp b/src/common/system_utils_posix.cpp index d71a0732434a8..ed47ff0595e23 100644 --- a/src/common/system_utils_posix.cpp +++ b/src/common/system_utils_posix.cpp @@ -125,4 +125,14 @@ void BreakDebugger() // See https://cs.chromium.org/chromium/src/base/debug/debugger_posix.cc abort(); } + +const char *GetExecutableExtension() +{ + return ""; +} + +char GetPathSeparator() +{ + return '/'; +} } // namespace angle diff --git a/src/common/system_utils_win.cpp b/src/common/system_utils_win.cpp index 48e26c96db59a..acfc628a1657a 100644 --- a/src/common/system_utils_win.cpp +++ b/src/common/system_utils_win.cpp @@ -166,4 +166,13 @@ void BreakDebugger() __debugbreak(); } +const char *GetExecutableExtension() +{ + return ".exe"; +} + +char GetPathSeparator() +{ + return '\\'; +} } // namespace angle diff --git a/src/tests/BUILD.gn b/src/tests/BUILD.gn index 34136f7beb8a7..81aa53c743f16 100644 --- a/src/tests/BUILD.gn +++ b/src/tests/BUILD.gn @@ -13,11 +13,18 @@ declare_args() { build_angle_gles1_conform_tests = false } -angle_executable("test_utils_unittest_helper") { - sources = test_utils_unittest_helper_sources +angle_test("test_utils_unittest_helper") { + standalone_harness = true + + sources = [ + "../../util/test_utils_unittest_helper.cpp", + "../../util/test_utils_unittest_helper.h", + "test_utils/angle_test_instantiate.h", + "test_utils/runner/TestSuite_unittest.cpp", + ] deps = [ - "${angle_root}:angle_common", + "$angle_root:angle_common", ] } diff --git a/src/tests/angle_deqp_tests_main.cpp b/src/tests/angle_deqp_tests_main.cpp index e72417f0a9cdf..0b24d7e75aac6 100644 --- a/src/tests/angle_deqp_tests_main.cpp +++ b/src/tests/angle_deqp_tests_main.cpp @@ -8,6 +8,8 @@ #include +#include "test_utils/runner/TestSuite.h" + // Defined in angle_deqp_gtest.cpp. Declared here so we don't need to make a header that we import // in Chromium. namespace angle @@ -18,7 +20,7 @@ void InitTestHarness(int *argc, char **argv); int main(int argc, char **argv) { angle::InitTestHarness(&argc, argv); - testing::InitGoogleTest(&argc, argv); + angle::TestSuite testSuite(&argc, argv); int rt = RUN_ALL_TESTS(); return rt; } diff --git a/src/tests/angle_end2end_tests_main.cpp b/src/tests/angle_end2end_tests_main.cpp index a6a2ca57b0b66..4972eaa26b7f1 100644 --- a/src/tests/angle_end2end_tests_main.cpp +++ b/src/tests/angle_end2end_tests_main.cpp @@ -5,13 +5,14 @@ // #include "gtest/gtest.h" +#include "test_utils/runner/TestSuite.h" void ANGLEProcessTestArgs(int *argc, char *argv[]); int main(int argc, char **argv) { + angle::TestSuite testSuite(&argc, argv); ANGLEProcessTestArgs(&argc, argv); - testing::InitGoogleTest(&argc, argv); int rt = RUN_ALL_TESTS(); return rt; } diff --git a/src/tests/angle_perftests_main.cpp b/src/tests/angle_perftests_main.cpp index 27f511ba4959e..804543eb4c4fc 100644 --- a/src/tests/angle_perftests_main.cpp +++ b/src/tests/angle_perftests_main.cpp @@ -9,13 +9,14 @@ #include +#include "test_utils/runner/TestSuite.h" + void ANGLEProcessPerfTestArgs(int *argc, char **argv); int main(int argc, char **argv) { + angle::TestSuite testSuite(&argc, argv); ANGLEProcessPerfTestArgs(&argc, argv); - testing::InitGoogleTest(&argc, argv); - testing::AddGlobalTestEnvironment(new testing::Environment()); int rt = RUN_ALL_TESTS(); return rt; } diff --git a/src/tests/angle_unittest_main.cpp b/src/tests/angle_unittest_main.cpp index 7bee7b831ceb6..72c200e5923a2 100644 --- a/src/tests/angle_unittest_main.cpp +++ b/src/tests/angle_unittest_main.cpp @@ -6,6 +6,7 @@ #include "GLSLANG/ShaderLang.h" #include "gtest/gtest.h" +#include "test_utils/runner/TestSuite.h" class CompilerTestEnvironment : public testing::Environment { @@ -29,8 +30,7 @@ class CompilerTestEnvironment : public testing::Environment int main(int argc, char **argv) { - testing::InitGoogleTest(&argc, argv); + angle::TestSuite testSuite(&argc, argv); testing::AddGlobalTestEnvironment(new CompilerTestEnvironment()); - int rt = RUN_ALL_TESTS(); - return rt; + return testSuite.run(); } diff --git a/src/tests/angle_unittests.gni b/src/tests/angle_unittests.gni index d6a86a37e1d75..590a409234679 100644 --- a/src/tests/angle_unittests.gni +++ b/src/tests/angle_unittests.gni @@ -44,91 +44,92 @@ angle_unittests_sources = [ "../libANGLE/renderer/ImageImpl_mock.h", "../libANGLE/renderer/TextureImpl_mock.h", "../libANGLE/renderer/TransformFeedbackImpl_mock.h", - "../tests/angle_unittests_utils.h", - "../tests/compiler_tests/API_test.cpp", - "../tests/compiler_tests/AppendixALimitations_test.cpp", - "../tests/compiler_tests/ARB_texture_rectangle_test.cpp", - "../tests/compiler_tests/AtomicCounter_test.cpp", - "../tests/compiler_tests/BufferVariables_test.cpp", - "../tests/compiler_tests/CollectVariables_test.cpp", - "../tests/compiler_tests/ConstantFolding_test.cpp", - "../tests/compiler_tests/ConstantFoldingNaN_test.cpp", - "../tests/compiler_tests/ConstantFoldingOverflow_test.cpp", - "../tests/compiler_tests/ConstructCompiler_test.cpp", - "../tests/compiler_tests/DebugShaderPrecision_test.cpp", - "../tests/compiler_tests/EmulateGLBaseVertexBaseInstance_test.cpp", - "../tests/compiler_tests/EmulateGLDrawID_test.cpp", - "../tests/compiler_tests/EmulateGLFragColorBroadcast_test.cpp", - "../tests/compiler_tests/ExpressionLimit_test.cpp", - "../tests/compiler_tests/EXT_YUV_target_test.cpp", - "../tests/compiler_tests/EXT_blend_func_extended_test.cpp", - "../tests/compiler_tests/EXT_frag_depth_test.cpp", - "../tests/compiler_tests/EXT_shader_texture_lod_test.cpp", - "../tests/compiler_tests/ExtensionDirective_test.cpp", - "../tests/compiler_tests/FloatLex_test.cpp", - "../tests/compiler_tests/FragDepth_test.cpp", - "../tests/compiler_tests/GLSLCompatibilityOutput_test.cpp", - "../tests/compiler_tests/GlFragDataNotModified_test.cpp", - "../tests/compiler_tests/GeometryShader_test.cpp", - "../tests/compiler_tests/ImmutableString_test.cpp", - "../tests/compiler_tests/InitOutputVariables_test.cpp", - "../tests/compiler_tests/IntermNode_test.cpp", - "../tests/compiler_tests/NV_draw_buffers_test.cpp", - "../tests/compiler_tests/OES_standard_derivatives_test.cpp", - "../tests/compiler_tests/Pack_Unpack_test.cpp", - "../tests/compiler_tests/PruneEmptyCases_test.cpp", - "../tests/compiler_tests/PruneEmptyDeclarations_test.cpp", - "../tests/compiler_tests/PrunePureLiteralStatements_test.cpp", - "../tests/compiler_tests/PruneUnusedFunctions_test.cpp", - "../tests/compiler_tests/QualificationOrderESSL31_test.cpp", - "../tests/compiler_tests/QualificationOrder_test.cpp", - "../tests/compiler_tests/RecordConstantPrecision_test.cpp", - "../tests/compiler_tests/RegenerateStructNames_test.cpp", - "../tests/compiler_tests/RemovePow_test.cpp", - "../tests/compiler_tests/RemoveUnreferencedVariables_test.cpp", - "../tests/compiler_tests/RewriteDoWhile_test.cpp", - "../tests/compiler_tests/SamplerMultisample_test.cpp", - "../tests/compiler_tests/ScalarizeVecAndMatConstructorArgs_test.cpp", - "../tests/compiler_tests/ShaderImage_test.cpp", - "../tests/compiler_tests/ShaderValidation_test.cpp", - "../tests/compiler_tests/ShaderVariable_test.cpp", - "../tests/compiler_tests/ShCompile_test.cpp", - "../tests/compiler_tests/TextureFunction_test.cpp", - "../tests/compiler_tests/Type_test.cpp", - "../tests/compiler_tests/TypeTracking_test.cpp", - "../tests/compiler_tests/UnfoldShortCircuitAST_test.cpp", - "../tests/compiler_tests/VariablePacker_test.cpp", - "../tests/compiler_tests/VectorizeVectorScalarArithmetic_test.cpp", - "../tests/compiler_tests/OVR_multiview_test.cpp", - "../tests/compiler_tests/OVR_multiview2_test.cpp", - "../tests/compiler_tests/WorkGroupSize_test.cpp", - "../tests/test_expectations/GPUTestExpectationsParser_unittest.cpp", - "../tests/preprocessor_tests/char_test.cpp", - "../tests/preprocessor_tests/comment_test.cpp", - "../tests/preprocessor_tests/define_test.cpp", - "../tests/preprocessor_tests/error_test.cpp", - "../tests/preprocessor_tests/extension_test.cpp", - "../tests/preprocessor_tests/identifier_test.cpp", - "../tests/preprocessor_tests/if_test.cpp", - "../tests/preprocessor_tests/input_test.cpp", - "../tests/preprocessor_tests/location_test.cpp", - "../tests/preprocessor_tests/MockDiagnostics.h", - "../tests/preprocessor_tests/MockDirectiveHandler.h", - "../tests/preprocessor_tests/number_test.cpp", - "../tests/preprocessor_tests/operator_test.cpp", - "../tests/preprocessor_tests/pragma_test.cpp", - "../tests/preprocessor_tests/PreprocessorTest.cpp", - "../tests/preprocessor_tests/PreprocessorTest.h", - "../tests/preprocessor_tests/space_test.cpp", - "../tests/preprocessor_tests/token_test.cpp", - "../tests/preprocessor_tests/version_test.cpp", - "../tests/test_utils/compiler_test.cpp", - "../tests/test_utils/compiler_test.h", - "../tests/test_utils/ConstantFoldingTest.h", - "../tests/test_utils/ConstantFoldingTest.cpp", - "../tests/test_utils/ShaderCompileTreeTest.h", - "../tests/test_utils/ShaderCompileTreeTest.cpp", - "../tests/test_utils/ShaderExtensionTest.h", + "angle_unittests_utils.h", + "compiler_tests/API_test.cpp", + "compiler_tests/AppendixALimitations_test.cpp", + "compiler_tests/ARB_texture_rectangle_test.cpp", + "compiler_tests/AtomicCounter_test.cpp", + "compiler_tests/BufferVariables_test.cpp", + "compiler_tests/CollectVariables_test.cpp", + "compiler_tests/ConstantFolding_test.cpp", + "compiler_tests/ConstantFoldingNaN_test.cpp", + "compiler_tests/ConstantFoldingOverflow_test.cpp", + "compiler_tests/ConstructCompiler_test.cpp", + "compiler_tests/DebugShaderPrecision_test.cpp", + "compiler_tests/EmulateGLBaseVertexBaseInstance_test.cpp", + "compiler_tests/EmulateGLDrawID_test.cpp", + "compiler_tests/EmulateGLFragColorBroadcast_test.cpp", + "compiler_tests/ExpressionLimit_test.cpp", + "compiler_tests/EXT_YUV_target_test.cpp", + "compiler_tests/EXT_blend_func_extended_test.cpp", + "compiler_tests/EXT_frag_depth_test.cpp", + "compiler_tests/EXT_shader_texture_lod_test.cpp", + "compiler_tests/ExtensionDirective_test.cpp", + "compiler_tests/FloatLex_test.cpp", + "compiler_tests/FragDepth_test.cpp", + "compiler_tests/GLSLCompatibilityOutput_test.cpp", + "compiler_tests/GlFragDataNotModified_test.cpp", + "compiler_tests/GeometryShader_test.cpp", + "compiler_tests/ImmutableString_test.cpp", + "compiler_tests/InitOutputVariables_test.cpp", + "compiler_tests/IntermNode_test.cpp", + "compiler_tests/NV_draw_buffers_test.cpp", + "compiler_tests/OES_standard_derivatives_test.cpp", + "compiler_tests/Pack_Unpack_test.cpp", + "compiler_tests/PruneEmptyCases_test.cpp", + "compiler_tests/PruneEmptyDeclarations_test.cpp", + "compiler_tests/PrunePureLiteralStatements_test.cpp", + "compiler_tests/PruneUnusedFunctions_test.cpp", + "compiler_tests/QualificationOrderESSL31_test.cpp", + "compiler_tests/QualificationOrder_test.cpp", + "compiler_tests/RecordConstantPrecision_test.cpp", + "compiler_tests/RegenerateStructNames_test.cpp", + "compiler_tests/RemovePow_test.cpp", + "compiler_tests/RemoveUnreferencedVariables_test.cpp", + "compiler_tests/RewriteDoWhile_test.cpp", + "compiler_tests/SamplerMultisample_test.cpp", + "compiler_tests/ScalarizeVecAndMatConstructorArgs_test.cpp", + "compiler_tests/ShaderImage_test.cpp", + "compiler_tests/ShaderValidation_test.cpp", + "compiler_tests/ShaderVariable_test.cpp", + "compiler_tests/ShCompile_test.cpp", + "compiler_tests/TextureFunction_test.cpp", + "compiler_tests/Type_test.cpp", + "compiler_tests/TypeTracking_test.cpp", + "compiler_tests/UnfoldShortCircuitAST_test.cpp", + "compiler_tests/VariablePacker_test.cpp", + "compiler_tests/VectorizeVectorScalarArithmetic_test.cpp", + "compiler_tests/OVR_multiview_test.cpp", + "compiler_tests/OVR_multiview2_test.cpp", + "compiler_tests/WorkGroupSize_test.cpp", + "test_expectations/GPUTestExpectationsParser_unittest.cpp", + "preprocessor_tests/char_test.cpp", + "preprocessor_tests/comment_test.cpp", + "preprocessor_tests/define_test.cpp", + "preprocessor_tests/error_test.cpp", + "preprocessor_tests/extension_test.cpp", + "preprocessor_tests/identifier_test.cpp", + "preprocessor_tests/if_test.cpp", + "preprocessor_tests/input_test.cpp", + "preprocessor_tests/location_test.cpp", + "preprocessor_tests/MockDiagnostics.h", + "preprocessor_tests/MockDirectiveHandler.h", + "preprocessor_tests/number_test.cpp", + "preprocessor_tests/operator_test.cpp", + "preprocessor_tests/pragma_test.cpp", + "preprocessor_tests/PreprocessorTest.cpp", + "preprocessor_tests/PreprocessorTest.h", + "preprocessor_tests/space_test.cpp", + "preprocessor_tests/token_test.cpp", + "preprocessor_tests/version_test.cpp", + "test_utils/angle_test_instantiate.h", + "test_utils/compiler_test.cpp", + "test_utils/compiler_test.h", + "test_utils/ConstantFoldingTest.h", + "test_utils/ConstantFoldingTest.cpp", + "test_utils/ShaderCompileTreeTest.h", + "test_utils/ShaderCompileTreeTest.cpp", + "test_utils/ShaderExtensionTest.h", "../../util/test_utils_unittest.cpp", "../../util/test_utils_unittest_helper.h", ] @@ -139,11 +140,6 @@ angle_unittests_hlsl_sources = [ "../tests/compiler_tests/UnrollFlatten_test.cpp", ] -test_utils_unittest_helper_sources = [ - "../../util/test_utils_unittest_helper.cpp", - "../../util/test_utils_unittest_helper.h", -] - if (is_android) { angle_unittests_sources += [ "../tests/compiler_tests/ImmutableString_test_ESSL_autogen.cpp" ] @@ -151,3 +147,8 @@ if (is_android) { angle_unittests_sources += [ "../tests/compiler_tests/ImmutableString_test_autogen.cpp" ] } + +if (!is_android && !is_fuchsia) { + angle_unittests_sources += + [ "../tests/test_utils/runner/TestSuite_unittest.cpp" ] +} diff --git a/src/tests/angle_white_box_tests_main.cpp b/src/tests/angle_white_box_tests_main.cpp index a940f4e227da0..77cb63b48d138 100644 --- a/src/tests/angle_white_box_tests_main.cpp +++ b/src/tests/angle_white_box_tests_main.cpp @@ -6,10 +6,11 @@ #include "gtest/gtest.h" #include "test_utils/ANGLETest.h" +#include "test_utils/runner/TestSuite.h" int main(int argc, char **argv) { - testing::InitGoogleTest(&argc, argv); + angle::TestSuite testSuite(&argc, argv); testing::AddGlobalTestEnvironment(new ANGLETestEnvironment()); int rt = RUN_ALL_TESTS(); return rt; diff --git a/src/tests/test_utils/ANGLETest.cpp b/src/tests/test_utils/ANGLETest.cpp index a00143a5c13ae..37f04e050bf4a 100644 --- a/src/tests/test_utils/ANGLETest.cpp +++ b/src/tests/test_utils/ANGLETest.cpp @@ -58,9 +58,7 @@ void TestPlatform_logError(PlatformMethods *platform, const char *errorMessage) GTEST_NONFATAL_FAILURE_(errorMessage); - // Print the stack and stop any crash handling to prevent duplicate reports. PrintStackBacktrace(); - TerminateCrashHandler(); } void TestPlatform_logWarning(PlatformMethods *platform, const char *warningMessage) @@ -481,8 +479,6 @@ void ANGLETestBase::ANGLETestSetUp() { mSetUpCalled = true; - InitCrashHandler(nullptr); - gDefaultPlatformMethods.overrideWorkaroundsD3D = TestPlatform_overrideWorkaroundsD3D; gDefaultPlatformMethods.overrideFeaturesVk = TestPlatform_overrideFeaturesVk; gDefaultPlatformMethods.logError = TestPlatform_logError; @@ -616,8 +612,6 @@ void ANGLETestBase::ANGLETestTearDown() mFixture->eglWindow->destroySurface(); } - TerminateCrashHandler(); - // Check for quit message Event myEvent; while (mFixture->osWindow->popEvent(&myEvent)) diff --git a/src/tests/test_utils/ANGLETest.h b/src/tests/test_utils/ANGLETest.h index 7dac71ed145be..70e30b7956433 100644 --- a/src/tests/test_utils/ANGLETest.h +++ b/src/tests/test_utils/ANGLETest.h @@ -625,14 +625,4 @@ bool IsGLExtensionRequestable(const std::string &extName); extern angle::PlatformMethods gDefaultPlatformMethods; -#define ANGLE_SKIP_TEST_IF(COND) \ - do \ - { \ - if (COND) \ - { \ - std::cout << "Test skipped: " #COND "." << std::endl; \ - return; \ - } \ - } while (0) - #endif // ANGLE_TESTS_ANGLE_TEST_H_ diff --git a/src/tests/test_utils/angle_test_instantiate.h b/src/tests/test_utils/angle_test_instantiate.h index a5c726688ed67..4aab41eeaa144 100644 --- a/src/tests/test_utils/angle_test_instantiate.h +++ b/src/tests/test_utils/angle_test_instantiate.h @@ -12,6 +12,8 @@ #include +#include "common/platform.h" + namespace angle { struct SystemInfo; @@ -40,6 +42,15 @@ bool IsIntel(); bool IsAMD(); bool IsNVIDIA(); +inline bool IsASan() +{ +#if defined(ANGLE_WITH_ASAN) + return true; +#else + return false; +#endif // defined(ANGLE_WITH_ASAN) +} + bool IsPlatformAvailable(const PlatformParameters ¶m); // This functions is used to filter which tests should be registered, @@ -208,4 +219,14 @@ extern std::string gSelectedConfig; extern bool gSeparateProcessPerConfig; } // namespace angle +#define ANGLE_SKIP_TEST_IF(COND) \ + do \ + { \ + if (COND) \ + { \ + std::cout << "Test skipped: " #COND "." << std::endl; \ + return; \ + } \ + } while (0) + #endif // ANGLE_TEST_INSTANTIATE_H_ diff --git a/src/tests/test_utils/runner/README.md b/src/tests/test_utils/runner/README.md new file mode 100644 index 0000000000000..d027a18fbd359 --- /dev/null +++ b/src/tests/test_utils/runner/README.md @@ -0,0 +1,47 @@ +# ANGLE Test Harness + +The ANGLE test harness is a harness around GoogleTest that provides functionality similar to the +[Chromium test harness][BaseTest]. It features: + + * splitting a test set into shards + * catching and reporting crashes and timeouts + * outputting to the Chromium [JSON test results format][JSONFormat] + * multi-process execution + +## Command-Line Arguments + +The ANGLE test harness accepts all standard GoogleTest arguments. The harness also accepts the +following additional command-line arguments: + + * `--shard-count` and `--shard-index` control the test sharding + * `--bot-mode` enables multi-process execution and test batching + * `--batch-size` limits the number of tests to run in each batch + * `--batch-timeout` limits the amount of time spent in each batch + * `--max-processes` limits the number of simuntaneous processes + * `--test-timeout` limits the amount of time spent in each test + * `--results-file` specifies a location for the JSON test result output + * `--results-directory` specifies a directory to write test results to + * `--filter-file` allows passing a larget `gtest_filter` via a file + +## Implementation Notes + + * The test harness only requires `angle_common` and `angle_util`. + * It does not depend on any Chromium browser code. This allows us to compile on other non-Clang platforms. + * It uses rapidjson to read and write JSON files. + * Timeouts are detected via a watchdog thread. + * Crashes are handled via ANGLE's test crash handling code. + * Currently it does not entirely support Android or Fuchsia. + * Test execution is not currently deterministic in multi-process mode. + * We capture stdout to output test failure reasons. + +See the source code for more details: [TestSuite.h](TestSuite.h) and [TestSuite.cpp](TestSuite.cpp). + +## Potential Areas of Improvement + + * Deterministic test execution. + * Using sockets to communicate with test children. Similar to dEQP's test harness. + * Closer integration with ANGLE's test expectations and system config libraries. + * Supporting a GoogleTest-free integration. + +[BaseTest]: https://chromium.googlesource.com/chromium/src/+/refs/heads/master/base/test/ +[JSONFormat]: https://chromium.googlesource.com/chromium/src/+/master/docs/testing/json_test_results_format.md diff --git a/src/tests/test_utils/runner/TestSuite.cpp b/src/tests/test_utils/runner/TestSuite.cpp new file mode 100644 index 0000000000000..97b5ad7689a76 --- /dev/null +++ b/src/tests/test_utils/runner/TestSuite.cpp @@ -0,0 +1,1187 @@ +// +// Copyright 2019 The ANGLE Project Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// TestSuite: +// Basic implementation of a test harness in ANGLE. + +#include "TestSuite.h" + +#include "common/debug.h" +#include "common/platform.h" +#include "common/system_utils.h" +#include "util/Timer.h" + +#include +#include + +#include +#include +#include +#include +#include + +// We directly call into a function to register the parameterized tests. This saves spinning up +// a subprocess with a new gtest filter. +#include "third_party/googletest/src/googletest/src/gtest-internal-inl.h" + +namespace js = rapidjson; + +namespace angle +{ +namespace +{ +constexpr char kTestTimeoutArg[] = "--test-timeout="; +constexpr char kFilterFileArg[] = "--filter-file="; +constexpr char kResultFileArg[] = "--results-file="; +constexpr int kDefaultTestTimeout = 10; +constexpr int kDefaultBatchSize = 1000; + +const char *ParseFlagValue(const char *flag, const char *argument) +{ + if (strstr(argument, flag) == argument) + { + return argument + strlen(flag); + } + + return nullptr; +} + +bool ParseIntArg(const char *flag, const char *argument, int *valueOut) +{ + const char *value = ParseFlagValue(flag, argument); + if (!value) + { + return false; + } + + char *end = nullptr; + const long longValue = strtol(value, &end, 10); + + if (*end != '\0') + { + printf("Error parsing integer flag value.\n"); + exit(1); + } + + if (longValue == LONG_MAX || longValue == LONG_MIN || static_cast(longValue) != longValue) + { + printf("Overflow when parsing integer flag value.\n"); + exit(1); + } + + *valueOut = static_cast(longValue); + return true; +} + +bool ParseFlag(const char *expected, const char *actual, bool *flagOut) +{ + if (strcmp(expected, actual) == 0) + { + *flagOut = true; + return true; + } + return false; +} + +bool ParseStringArg(const char *flag, const char *argument, std::string *valueOut) +{ + const char *value = ParseFlagValue(flag, argument); + if (!value) + { + return false; + } + + *valueOut = value; + return true; +} + +void DeleteArg(int *argc, char **argv, int argIndex) +{ + // Shift the remainder of the argv list left by one. Note that argv has (*argc + 1) elements, + // the last one always being NULL. The following loop moves the trailing NULL element as well. + for (int index = argIndex; index < *argc; ++index) + { + argv[index] = argv[index + 1]; + } + (*argc)--; +} + +void AddArg(int *argc, char **argv, const char *arg) +{ + // This unsafe const_cast is necessary to work around gtest limitations. + argv[*argc] = const_cast(arg); + argv[*argc + 1] = nullptr; + (*argc)++; +} + +const char *ResultTypeToString(TestResultType type) +{ + switch (type) + { + case TestResultType::Crash: + return "CRASH"; + case TestResultType::Fail: + return "FAIL"; + case TestResultType::Pass: + return "PASS"; + case TestResultType::Skip: + return "SKIP"; + case TestResultType::Timeout: + return "TIMEOUT"; + case TestResultType::Unknown: + return "UNKNOWN"; + } +} + +TestResultType GetResultTypeFromString(const std::string &str) +{ + if (str == "CRASH") + return TestResultType::Crash; + if (str == "FAIL") + return TestResultType::Fail; + if (str == "PASS") + return TestResultType::Pass; + if (str == "SKIP") + return TestResultType::Skip; + if (str == "TIMEOUT") + return TestResultType::Timeout; + return TestResultType::Unknown; +} + +js::Value ResultTypeToJSString(TestResultType type, js::Document::AllocatorType *allocator) +{ + js::Value jsName; + jsName.SetString(ResultTypeToString(type), *allocator); + return jsName; +} + +// Writes out a TestResults to the Chromium JSON Test Results format. +// https://chromium.googlesource.com/chromium/src.git/+/master/docs/testing/json_test_results_format.md +void WriteTestResults(bool interrupted, + const TestResults &testResults, + const std::string &outputFile, + const char *testSuiteName) +{ + time_t ltime; + time(<ime); + struct tm *timeinfo = gmtime(<ime); + ltime = mktime(timeinfo); + + uint64_t secondsSinceEpoch = static_cast(ltime); + + js::Document doc; + doc.SetObject(); + + js::Document::AllocatorType &allocator = doc.GetAllocator(); + + doc.AddMember("interrupted", interrupted, allocator); + doc.AddMember("path_delimiter", ".", allocator); + doc.AddMember("version", 3, allocator); + doc.AddMember("seconds_since_epoch", secondsSinceEpoch, allocator); + + js::Value testSuite; + testSuite.SetObject(); + + std::map counts; + + for (const auto &resultIter : testResults.results) + { + const TestIdentifier &id = resultIter.first; + const TestResult &result = resultIter.second; + + js::Value jsResult; + jsResult.SetObject(); + + counts[result.type]++; + + jsResult.AddMember("expected", "PASS", allocator); + jsResult.AddMember("actual", ResultTypeToJSString(result.type, &allocator), allocator); + + js::Value times; + times.SetArray(); + times.PushBack(result.elapsedTimeSeconds, allocator); + + jsResult.AddMember("times", times, allocator); + + char testName[500]; + id.sprintfName(testName); + js::Value jsName; + jsName.SetString(testName, allocator); + + testSuite.AddMember(jsName, jsResult, allocator); + } + + js::Value numFailuresByType; + numFailuresByType.SetObject(); + + for (const auto &countIter : counts) + { + TestResultType type = countIter.first; + uint32_t count = countIter.second; + + js::Value jsCount(count); + numFailuresByType.AddMember(ResultTypeToJSString(type, &allocator), jsCount, allocator); + } + + doc.AddMember("num_failures_by_type", numFailuresByType, allocator); + + js::Value tests; + tests.SetObject(); + tests.AddMember(js::StringRef(testSuiteName), testSuite, allocator); + + doc.AddMember("tests", tests, allocator); + + printf("Writing test results to %s\n", outputFile.c_str()); + + FILE *fp = fopen(outputFile.c_str(), "w"); + + constexpr size_t kBufferSize = 0xFFFF; + std::vector writeBuffer(kBufferSize); + js::FileWriteStream os(fp, writeBuffer.data(), kBufferSize); + js::PrettyWriter writer(os); + doc.Accept(writer); + + fclose(fp); +} + +void UpdateCurrentTestResult(const testing::TestResult &resultIn, TestResults *resultsOut) +{ + TestResult &resultOut = resultsOut->results[resultsOut->currentTest]; + + // Note: Crashes and Timeouts are detected by the crash handler and a watchdog thread. + if (resultIn.Skipped()) + { + resultOut.type = TestResultType::Skip; + } + else if (resultIn.Failed()) + { + resultOut.type = TestResultType::Fail; + } + else + { + resultOut.type = TestResultType::Pass; + } + + resultOut.elapsedTimeSeconds = resultsOut->currentTestTimer.getElapsedTime(); +} + +TestIdentifier GetTestIdentifier(const testing::TestInfo &testInfo) +{ + return {testInfo.test_suite_name(), testInfo.name()}; +} + +class TestEventListener : public testing::EmptyTestEventListener +{ + public: + // Note: TestResults is owned by the TestSuite. It should outlive TestEventListener. + TestEventListener(const std::string &outputFile, + const char *testSuiteName, + TestResults *testResults) + : mResultsFile(outputFile), mTestSuiteName(testSuiteName), mTestResults(testResults) + {} + + void OnTestStart(const testing::TestInfo &testInfo) override + { + std::lock_guard guard(mTestResults->currentTestMutex); + mTestResults->currentTest = GetTestIdentifier(testInfo); + mTestResults->currentTestTimer.start(); + } + + void OnTestEnd(const testing::TestInfo &testInfo) override + { + std::lock_guard guard(mTestResults->currentTestMutex); + mTestResults->currentTestTimer.stop(); + const testing::TestResult &resultIn = *testInfo.result(); + UpdateCurrentTestResult(resultIn, mTestResults); + mTestResults->currentTest = TestIdentifier(); + } + + void OnTestProgramEnd(const testing::UnitTest &testProgramInfo) override + { + std::lock_guard guard(mTestResults->currentTestMutex); + mTestResults->allDone = true; + WriteTestResults(false, *mTestResults, mResultsFile, mTestSuiteName); + } + + private: + std::string mResultsFile; + const char *mTestSuiteName; + TestResults *mTestResults; +}; + +using TestIdentifierFilter = std::function; + +std::vector FilterTests(std::map *fileLinesOut, + TestIdentifierFilter filter) +{ + std::vector tests; + + const testing::UnitTest &testProgramInfo = *testing::UnitTest::GetInstance(); + for (int suiteIndex = 0; suiteIndex < testProgramInfo.total_test_suite_count(); ++suiteIndex) + { + const testing::TestSuite &testSuite = *testProgramInfo.GetTestSuite(suiteIndex); + for (int testIndex = 0; testIndex < testSuite.total_test_count(); ++testIndex) + { + const testing::TestInfo &testInfo = *testSuite.GetTestInfo(testIndex); + TestIdentifier id = GetTestIdentifier(testInfo); + if (filter(id)) + { + tests.emplace_back(id); + + if (fileLinesOut) + { + (*fileLinesOut)[id] = {testInfo.file(), testInfo.line()}; + } + } + } + } + + return tests; +} + +std::vector GetFilteredTests(std::map *fileLinesOut) +{ + TestIdentifierFilter gtestIDFilter = [](const TestIdentifier &id) { + return testing::internal::UnitTestOptions::FilterMatchesTest(id.testSuiteName, id.testName); + }; + + return FilterTests(fileLinesOut, gtestIDFilter); +} + +std::vector GetCompiledInTests(std::map *fileLinesOut) +{ + TestIdentifierFilter passthroughFilter = [](const TestIdentifier &id) { return true; }; + return FilterTests(fileLinesOut, passthroughFilter); +} + +std::vector GetShardTests(int shardIndex, + int shardCount, + std::map *fileLinesOut) +{ + std::vector allTests = GetCompiledInTests(fileLinesOut); + std::vector shardTests; + + for (int testIndex = shardIndex; testIndex < static_cast(allTests.size()); + testIndex += shardCount) + { + shardTests.emplace_back(allTests[testIndex]); + } + + return shardTests; +} + +std::string GetTestFilter(const std::vector &tests) +{ + std::stringstream filterStream; + + filterStream << "--gtest_filter="; + + for (size_t testIndex = 0; testIndex < tests.size(); ++testIndex) + { + if (testIndex != 0) + { + filterStream << ":"; + } + + filterStream << tests[testIndex]; + } + + return filterStream.str(); +} + +std::string ParseTestSuiteName(const char *executable) +{ + const char *baseNameStart = strrchr(executable, GetPathSeparator()); + if (!baseNameStart) + { + baseNameStart = executable; + } + else + { + baseNameStart++; + } + + const char *suffix = GetExecutableExtension(); + size_t suffixLen = strlen(suffix); + if (suffixLen == 0) + { + return baseNameStart; + } + + const char *baseNameSuffix = strstr(baseNameStart, suffix); + ASSERT(baseNameSuffix == (baseNameStart + strlen(baseNameStart) - suffixLen)); + return std::string(baseNameStart, baseNameSuffix); +} + +bool GetTestResultsFromJSON(const js::Document &document, TestResults *resultsOut) +{ + if (!document.HasMember("tests") || !document["tests"].IsObject()) + { + return false; + } + + const js::Value::ConstObject &tests = document["tests"].GetObject(); + if (tests.MemberCount() != 1) + { + return false; + } + + const js::Value::Member &suite = *tests.MemberBegin(); + if (!suite.value.IsObject()) + { + return false; + } + + const js::Value::ConstObject &actual = suite.value.GetObject(); + + for (auto iter = actual.MemberBegin(); iter != actual.MemberEnd(); ++iter) + { + // Get test identifier. + const js::Value &name = iter->name; + if (!name.IsString()) + { + return false; + } + + TestIdentifier id; + if (!TestIdentifier::ParseFromString(name.GetString(), &id)) + { + return false; + } + + // Get test result. + const js::Value &value = iter->value; + if (!value.IsObject()) + { + return false; + } + + const js::Value::ConstObject &obj = value.GetObject(); + if (!obj.HasMember("expected") || !obj.HasMember("actual")) + { + return false; + } + + const js::Value &expected = obj["expected"]; + const js::Value &actual = obj["actual"]; + + if (!expected.IsString() || !actual.IsString()) + { + return false; + } + + const std::string expectedStr = expected.GetString(); + const std::string actualStr = actual.GetString(); + + if (expectedStr != "PASS") + { + return false; + } + + TestResultType resultType = GetResultTypeFromString(actualStr); + if (resultType == TestResultType::Unknown) + { + return false; + } + + double elapsedTimeSeconds = 0.0; + if (obj.HasMember("times")) + { + const js::Value × = obj["times"]; + if (!times.IsArray()) + { + return false; + } + + const js::Value::ConstArray ×Array = times.GetArray(); + if (timesArray.Size() != 1 || !timesArray[0].IsDouble()) + { + return false; + } + + elapsedTimeSeconds = timesArray[0].GetDouble(); + } + + TestResult &result = resultsOut->results[id]; + result.elapsedTimeSeconds = elapsedTimeSeconds; + result.type = resultType; + } + + return true; +} + +bool MergeTestResults(const TestResults &input, TestResults *output) +{ + for (const auto &resultsIter : input.results) + { + const TestIdentifier &id = resultsIter.first; + const TestResult &inputResult = resultsIter.second; + TestResult &outputResult = output->results[id]; + + // This should probably handle situations where a test is run more than once. + if (inputResult.type != TestResultType::Skip) + { + if (outputResult.type != TestResultType::Skip) + { + printf("Warning: duplicate entry for %s.%s.\n", id.testSuiteName.c_str(), + id.testName.c_str()); + return false; + } + + outputResult.elapsedTimeSeconds = inputResult.elapsedTimeSeconds; + outputResult.type = inputResult.type; + } + } + + return true; +} + +void PrintTestOutputSnippet(const TestIdentifier &id, + const TestResult &result, + const std::string &fullOutput) +{ + std::stringstream nameStream; + nameStream << id; + std::string fullName = nameStream.str(); + + size_t runPos = fullOutput.find(std::string("[ RUN ] ") + fullName); + if (runPos == std::string::npos) + { + printf("Cannot locate test output snippet.\n"); + return; + } + + size_t endPos = fullOutput.find(std::string("[ FAILED ] ") + fullName, runPos); + // Only clip the snippet to the "OK" message if the test really + // succeeded. It still might have e.g. crashed after printing it. + if (endPos == std::string::npos && result.type == TestResultType::Pass) + { + endPos = fullOutput.find(std::string("[ OK ] ") + fullName, runPos); + } + if (endPos != std::string::npos) + { + size_t newline_pos = fullOutput.find("\n", endPos); + if (newline_pos != std::string::npos) + endPos = newline_pos + 1; + } + + std::cout << "\n"; + if (endPos != std::string::npos) + { + std::cout << fullOutput.substr(runPos, endPos - runPos); + } + else + { + std::cout << fullOutput.substr(runPos); + } + std::cout << "\n"; +} +} // namespace + +TestIdentifier::TestIdentifier() = default; + +TestIdentifier::TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn) + : testSuiteName(suiteNameIn), testName(nameIn) +{} + +TestIdentifier::TestIdentifier(const TestIdentifier &other) = default; + +TestIdentifier::~TestIdentifier() = default; + +TestIdentifier &TestIdentifier::operator=(const TestIdentifier &other) = default; + +void TestIdentifier::sprintfName(char *outBuffer) const +{ + sprintf(outBuffer, "%s.%s", testSuiteName.c_str(), testName.c_str()); +} + +// static +bool TestIdentifier::ParseFromString(const std::string &str, TestIdentifier *idOut) +{ + size_t separator = str.find("."); + if (separator == std::string::npos) + { + return false; + } + + idOut->testSuiteName = str.substr(0, separator); + idOut->testName = str.substr(separator + 1, str.length() - separator - 1); + return true; +} + +TestResults::TestResults() = default; + +TestResults::~TestResults() = default; + +ProcessInfo::ProcessInfo() = default; + +ProcessInfo &ProcessInfo::operator=(ProcessInfo &&rhs) +{ + process = std::move(rhs.process); + testsInBatch = std::move(rhs.testsInBatch); + resultsFileName = std::move(rhs.resultsFileName); + filterFileName = std::move(rhs.filterFileName); + commandLine = std::move(rhs.commandLine); + return *this; +} + +ProcessInfo::~ProcessInfo() = default; + +ProcessInfo::ProcessInfo(ProcessInfo &&other) +{ + *this = std::move(other); +} + +TestSuite::TestSuite(int *argc, char **argv) + : mShardCount(-1), + mShardIndex(-1), + mBotMode(false), + mBatchSize(kDefaultBatchSize), + mCurrentResultCount(0), + mTotalResultCount(0), + mMaxProcesses(NumberOfProcessors()), + mTestTimeout(kDefaultTestTimeout), + mBatchTimeout(60) +{ + bool hasFilter = false; + +#if defined(ANGLE_PLATFORM_WINDOWS) + testing::GTEST_FLAG(catch_exceptions) = false; +#endif + + // Note that the crash callback must be owned and not use global constructors. + mCrashCallback = [this]() { onCrashOrTimeout(TestResultType::Crash); }; + InitCrashHandler(&mCrashCallback); + + if (*argc <= 0) + { + printf("Missing test arguments.\n"); + exit(1); + } + + mTestExecutableName = argv[0]; + mTestSuiteName = ParseTestSuiteName(mTestExecutableName.c_str()); + + for (int argIndex = 1; argIndex < *argc;) + { + if (parseSingleArg(argv[argIndex])) + { + DeleteArg(argc, argv, argIndex); + continue; + } + + if (ParseFlagValue("--gtest_filter=", argv[argIndex])) + { + hasFilter = true; + } + else + { + mGoogleTestCommandLineArgs.push_back(argv[argIndex]); + } + ++argIndex; + } + + if ((mShardIndex >= 0) != (mShardCount > 1)) + { + printf("Shard index and shard count must be specified together.\n"); + exit(1); + } + + if (!mFilterFile.empty()) + { + if (hasFilter) + { + printf("Cannot use gtest_filter in conjunction with a filter file.\n"); + exit(1); + } + + if (mShardCount > 0) + { + printf("Cannot use filter file in conjunction with sharding parameters.\n"); + exit(1); + } + + uint32_t fileSize = 0; + if (!GetFileSize(mFilterFile.c_str(), &fileSize)) + { + printf("Error getting filter file size: %s\n", mFilterFile.c_str()); + exit(1); + } + + std::vector fileContents(fileSize + 1, 0); + if (!ReadEntireFileToString(mFilterFile.c_str(), fileContents.data(), fileSize)) + { + printf("Error loading filter file: %s\n", mFilterFile.c_str()); + exit(1); + } + mFilterString.assign(fileContents.data()); + + if (mFilterString.substr(0, strlen("--gtest_filter=")) != std::string("--gtest_filter=")) + { + printf("Filter file must start with \"--gtest_filter=\"."); + exit(1); + } + + // Note that we only add a filter string if we previously deleted a shader filter file + // argument. So we will have space for the new filter string in argv. + AddArg(argc, argv, mFilterString.c_str()); + } + + // Call into gtest internals to force parameterized test name registration. + // TODO(jmadill): Clean this up so we don't need to call it. + testing::internal::UnitTestImpl *impl = testing::internal::GetUnitTestImpl(); + impl->RegisterParameterizedTests(); + + if (mShardCount > 0) + { + if (hasFilter) + { + printf("Cannot use gtest_filter in conjunction with sharding parameters.\n"); + exit(1); + } + + mTestQueue = GetShardTests(mShardIndex, mShardCount, &mTestFileLines); + mFilterString = GetTestFilter(mTestQueue); + + // Note that we only add a filter string if we previously deleted a shader index/count + // argument. So we will have space for the new filter string in argv. + AddArg(argc, argv, mFilterString.c_str()); + } + + testing::InitGoogleTest(argc, argv); + + if (mShardCount <= 0) + { + mTestQueue = GetFilteredTests(&mTestFileLines); + } + + mTotalResultCount = mTestQueue.size(); + + if ((mBotMode || !mResultsDirectory.empty()) && mResultsFile.empty()) + { + // Create a default output file in bot mode. + mResultsFile = "output.json"; + } + + if (!mResultsDirectory.empty()) + { + std::stringstream resultFileName; + resultFileName << mResultsDirectory << GetPathSeparator() << mResultsFile; + mResultsFile = resultFileName.str(); + } + + if (!mResultsFile.empty()) + { + testing::TestEventListeners &listeners = testing::UnitTest::GetInstance()->listeners(); + listeners.Append( + new TestEventListener(mResultsFile, mTestSuiteName.c_str(), &mTestResults)); + + std::vector testList = GetFilteredTests(nullptr); + + for (const TestIdentifier &id : testList) + { + mTestResults.results[id].type = TestResultType::Skip; + } + } +} + +TestSuite::~TestSuite() +{ + if (mWatchdogThread.joinable()) + { + mWatchdogThread.detach(); + } + TerminateCrashHandler(); +} + +bool TestSuite::parseSingleArg(const char *argument) +{ + return (ParseIntArg("--shard-count=", argument, &mShardCount) || + ParseIntArg("--shard-index=", argument, &mShardIndex) || + ParseIntArg("--batch-size=", argument, &mBatchSize) || + ParseIntArg("--max-processes=", argument, &mMaxProcesses) || + ParseIntArg(kTestTimeoutArg, argument, &mTestTimeout) || + ParseIntArg("--batch-timeout=", argument, &mBatchTimeout) || + ParseStringArg("--results-directory=", argument, &mResultsDirectory) || + ParseStringArg(kResultFileArg, argument, &mResultsFile) || + ParseStringArg(kFilterFileArg, argument, &mFilterFile) || + ParseFlag("--bot-mode", argument, &mBotMode)); +} + +void TestSuite::onCrashOrTimeout(TestResultType crashOrTimeout) +{ + if (mTestResults.currentTest.valid()) + { + TestResult &result = mTestResults.results[mTestResults.currentTest]; + result.type = crashOrTimeout; + result.elapsedTimeSeconds = mTestResults.currentTestTimer.getElapsedTime(); + } + + if (mResultsFile.empty()) + { + printf("No results file specified.\n"); + return; + } + + WriteTestResults(true, mTestResults, mResultsFile, mTestSuiteName.c_str()); +} + +bool TestSuite::launchChildTestProcess(const std::vector &testsInBatch) +{ + constexpr uint32_t kMaxPath = 1000; + + // Create a temporary file to store the test list + ProcessInfo processInfo; + + char filterBuffer[kMaxPath] = {}; + if (!CreateTemporaryFile(filterBuffer, kMaxPath)) + { + std::cerr << "Error creating temporary file for test list.\n"; + return false; + } + processInfo.filterFileName.assign(filterBuffer); + + std::string filterString = GetTestFilter(testsInBatch); + + FILE *fp = fopen(processInfo.filterFileName.c_str(), "w"); + if (!fp) + { + std::cerr << "Error opening temporary file for test list.\n"; + return false; + } + fprintf(fp, "%s", filterString.c_str()); + fclose(fp); + + std::string filterFileArg = kFilterFileArg + processInfo.filterFileName; + + // Create a temporary file to store the test output. + char resultsBuffer[kMaxPath] = {}; + if (!CreateTemporaryFile(resultsBuffer, kMaxPath)) + { + std::cerr << "Error creating temporary file for test list.\n"; + return false; + } + processInfo.resultsFileName.assign(resultsBuffer); + + std::string resultsFileArg = kResultFileArg + processInfo.resultsFileName; + + // Construct commandline for child process. + std::vector args; + + args.push_back(mTestExecutableName.c_str()); + args.push_back(filterFileArg.c_str()); + args.push_back(resultsFileArg.c_str()); + + for (const std::string &arg : mGoogleTestCommandLineArgs) + { + args.push_back(arg.c_str()); + } + + std::string timeoutStr; + if (mTestTimeout != kDefaultTestTimeout) + { + std::stringstream timeoutStream; + timeoutStream << kTestTimeoutArg << mTestTimeout; + timeoutStr = timeoutStream.str(); + args.push_back(timeoutStr.c_str()); + } + + // Launch child process and wait for completion. + processInfo.process = LaunchProcess(args, true, true); + + if (!processInfo.process->started()) + { + std::cerr << "Error launching child process.\n"; + return false; + } + + std::stringstream commandLineStr; + for (const char *arg : args) + { + commandLineStr << arg << " "; + } + + processInfo.commandLine = commandLineStr.str(); + processInfo.testsInBatch = testsInBatch; + mCurrentProcesses.emplace_back(std::move(processInfo)); + return true; +} + +bool TestSuite::finishProcess(ProcessInfo *processInfo) +{ + // Get test results and merge into master list. + TestResults batchResults; + + if (!GetTestResultsFromFile(processInfo->resultsFileName.c_str(), &batchResults)) + { + std::cerr << "Error reading test results from child process.\n"; + return false; + } + + if (!MergeTestResults(batchResults, &mTestResults)) + { + std::cerr << "Error merging batch test results.\n"; + return false; + } + + // Process results and print unexpected errors. + for (const auto &resultIter : batchResults.results) + { + const TestIdentifier &id = resultIter.first; + const TestResult &result = resultIter.second; + + // Skip results aren't procesed since they're added back to the test queue below. + if (result.type == TestResultType::Skip) + { + continue; + } + + mCurrentResultCount++; + printf("[%d/%d] %s.%s", mCurrentResultCount, mTotalResultCount, id.testSuiteName.c_str(), + id.testName.c_str()); + + if (result.type == TestResultType::Pass) + { + printf(" (%g ms)\n", result.elapsedTimeSeconds * 1000.0); + } + else + { + printf(" (%s)\n", ResultTypeToString(result.type)); + + const std::string &batchStdout = processInfo->process->getStdout(); + PrintTestOutputSnippet(id, result, batchStdout); + } + } + + // On unexpected exit, re-queue any unfinished tests. + if (processInfo->process->getExitCode() != 0) + { + for (const auto &resultIter : batchResults.results) + { + const TestIdentifier &id = resultIter.first; + const TestResult &result = resultIter.second; + + if (result.type == TestResultType::Skip) + { + mTestQueue.emplace_back(id); + } + } + } + + // Clean up any dirty temporary files. + for (const std::string &tempFile : {processInfo->filterFileName, processInfo->resultsFileName}) + { + // Note: we should be aware that this cleanup won't happen if the harness itself crashes. + // If this situation comes up in the future we should add crash cleanup to the harness. + if (!angle::DeleteFile(tempFile.c_str())) + { + std::cerr << "Warning: Error cleaning up temp file: " << tempFile << "\n"; + } + } + + processInfo->process.reset(); + return true; +} + +int TestSuite::run() +{ + // Run tests serially. + if (!mBotMode) + { + startWatchdog(); + return RUN_ALL_TESTS(); + } + + constexpr double kIdleMessageTimeout = 5.0; + + Timer messageTimer; + messageTimer.start(); + + while (!mTestQueue.empty() || !mCurrentProcesses.empty()) + { + bool progress = false; + + // Spawn a process if needed and possible. + while (static_cast(mCurrentProcesses.size()) < mMaxProcesses && !mTestQueue.empty()) + { + int numTests = std::min(mTestQueue.size(), mBatchSize); + + std::vector testsInBatch; + testsInBatch.assign(mTestQueue.begin(), mTestQueue.begin() + numTests); + mTestQueue.erase(mTestQueue.begin(), mTestQueue.begin() + numTests); + + if (!launchChildTestProcess(testsInBatch)) + { + return 1; + } + + progress = true; + } + + // Check for process completion. + for (auto processIter = mCurrentProcesses.begin(); processIter != mCurrentProcesses.end();) + { + ProcessInfo &processInfo = *processIter; + if (processInfo.process->finished()) + { + if (!finishProcess(&processInfo)) + { + return 1; + } + processIter = mCurrentProcesses.erase(processIter); + progress = true; + } + else if (processInfo.process->getElapsedTimeSeconds() > mBatchTimeout) + { + // Terminate the process and record timeouts for the batch. + // Because we can't determine which sub-test caused a timeout, record the whole + // batch as a timeout failure. Can be improved by using socket message passing. + if (!processInfo.process->kill()) + { + return 1; + } + for (const TestIdentifier &testIdentifier : processInfo.testsInBatch) + { + // Because the whole batch failed we can't know how long each test took. + mTestResults.results[testIdentifier].type = TestResultType::Timeout; + } + + processIter = mCurrentProcesses.erase(processIter); + progress = true; + } + else + { + processIter++; + } + } + + if (!progress && messageTimer.getElapsedTime() > kIdleMessageTimeout) + { + for (const ProcessInfo &processInfo : mCurrentProcesses) + { + double processTime = processInfo.process->getElapsedTimeSeconds(); + if (processTime > kIdleMessageTimeout) + { + printf("Running for %d seconds: %s\n", static_cast(processTime), + processInfo.commandLine.c_str()); + } + } + + messageTimer.start(); + } + + // Sleep briefly and continue. + angle::Sleep(10); + } + + // Dump combined results. + WriteTestResults(true, mTestResults, mResultsFile, mTestSuiteName.c_str()); + + return printFailuresAndReturnCount() == 0; +} + +int TestSuite::printFailuresAndReturnCount() const +{ + std::vector failures; + + for (const auto &resultIter : mTestResults.results) + { + const TestIdentifier &id = resultIter.first; + const TestResult &result = resultIter.second; + + if (result.type != TestResultType::Pass) + { + const FileLine &fileLine = mTestFileLines.find(id)->second; + + std::stringstream failureMessage; + failureMessage << id << " (" << fileLine.file << ":" << fileLine.line << ") (" + << ResultTypeToString(result.type) << ")"; + failures.emplace_back(failureMessage.str()); + } + } + + if (failures.empty()) + return 0; + + printf("%zu test%s failed:\n", failures.size(), failures.size() > 1 ? "s" : ""); + for (const std::string &failure : failures) + { + printf(" %s\n", failure.c_str()); + } + + return static_cast(failures.size()); +} + +void TestSuite::startWatchdog() +{ + auto watchdogMain = [this]() { + do + { + { + std::lock_guard guard(mTestResults.currentTestMutex); + if (mTestResults.currentTestTimer.getElapsedTime() > + static_cast(mTestTimeout)) + { + onCrashOrTimeout(TestResultType::Timeout); + exit(2); + } + + if (mTestResults.allDone) + return; + } + + angle::Sleep(1000); + } while (true); + }; + mWatchdogThread = std::thread(watchdogMain); +} + +bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut) +{ + std::ifstream ifs(fileName); + if (!ifs.is_open()) + { + std::cerr << "Error opening " << fileName << "\n"; + return false; + } + + js::IStreamWrapper ifsWrapper(ifs); + js::Document document; + document.ParseStream(ifsWrapper); + + if (document.HasParseError()) + { + std::cerr << "Parse error reading JSON document: " << document.GetParseError() << "\n"; + return false; + } + + if (!GetTestResultsFromJSON(document, resultsOut)) + { + std::cerr << "Error getting test results from JSON.\n"; + return false; + } + + return true; +} + +const char *TestResultTypeToString(TestResultType type) +{ + switch (type) + { + case TestResultType::Crash: + return "Crash"; + case TestResultType::Fail: + return "Fail"; + case TestResultType::Skip: + return "Skip"; + case TestResultType::Pass: + return "Pass"; + case TestResultType::Timeout: + return "Timeout"; + case TestResultType::Unknown: + return "Unknown"; + } +} +} // namespace angle diff --git a/src/tests/test_utils/runner/TestSuite.h b/src/tests/test_utils/runner/TestSuite.h new file mode 100644 index 0000000000000..f933932fedfe8 --- /dev/null +++ b/src/tests/test_utils/runner/TestSuite.h @@ -0,0 +1,153 @@ +// +// Copyright 2019 The ANGLE Project Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// TestSuite: +// Basic implementation of a test harness in ANGLE. + +#include +#include +#include +#include +#include + +#include "util/test_utils.h" + +namespace angle +{ +struct TestIdentifier +{ + TestIdentifier(); + TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn); + TestIdentifier(const TestIdentifier &other); + ~TestIdentifier(); + + TestIdentifier &operator=(const TestIdentifier &other); + + static bool ParseFromString(const std::string &str, TestIdentifier *idOut); + + bool valid() const { return !testName.empty(); } + void sprintfName(char *outBuffer) const; + + std::string testSuiteName; + std::string testName; +}; + +inline bool operator<(const TestIdentifier &a, const TestIdentifier &b) +{ + return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName); +} + +inline bool operator==(const TestIdentifier &a, const TestIdentifier &b) +{ + return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName); +} + +inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id) +{ + return os << id.testSuiteName << "." << id.testName; +} + +enum class TestResultType +{ + Crash, + Fail, + Skip, + Pass, + Timeout, + Unknown, +}; + +const char *TestResultTypeToString(TestResultType type); + +struct TestResult +{ + TestResultType type = TestResultType::Skip; + double elapsedTimeSeconds = 0.0; +}; + +inline bool operator==(const TestResult &a, const TestResult &b) +{ + return a.type == b.type; +} + +inline std::ostream &operator<<(std::ostream &os, const TestResult &result) +{ + return os << TestResultTypeToString(result.type); +} + +struct TestResults +{ + TestResults(); + ~TestResults(); + + std::map results; + std::mutex currentTestMutex; + TestIdentifier currentTest; + Timer currentTestTimer; + bool allDone = false; +}; + +struct FileLine +{ + const char *file; + int line; +}; + +struct ProcessInfo : angle::NonCopyable +{ + ProcessInfo(); + ~ProcessInfo(); + ProcessInfo(ProcessInfo &&other); + ProcessInfo &operator=(ProcessInfo &&rhs); + + ProcessHandle process; + std::vector testsInBatch; + std::string resultsFileName; + std::string filterFileName; + std::string commandLine; +}; + +class TestSuite +{ + public: + TestSuite(int *argc, char **argv); + ~TestSuite(); + + int run(); + void onCrashOrTimeout(TestResultType crashOrTimeout); + + private: + bool parseSingleArg(const char *argument); + bool launchChildTestProcess(const std::vector &testsInBatch); + bool finishProcess(ProcessInfo *processInfo); + int printFailuresAndReturnCount() const; + void startWatchdog(); + + std::string mTestExecutableName; + std::string mTestSuiteName; + std::vector mTestQueue; + std::string mFilterString; + std::string mFilterFile; + std::string mResultsDirectory; + std::string mResultsFile; + int mShardCount; + int mShardIndex; + angle::CrashCallback mCrashCallback; + TestResults mTestResults; + bool mBotMode; + int mBatchSize; + int mCurrentResultCount; + int mTotalResultCount; + int mMaxProcesses; + int mTestTimeout; + int mBatchTimeout; + std::vector mGoogleTestCommandLineArgs; + std::map mTestFileLines; + std::vector mCurrentProcesses; + std::thread mWatchdogThread; +}; + +bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut); +} // namespace angle diff --git a/src/tests/test_utils/runner/TestSuite_unittest.cpp b/src/tests/test_utils/runner/TestSuite_unittest.cpp new file mode 100644 index 0000000000000..9500e840ae950 --- /dev/null +++ b/src/tests/test_utils/runner/TestSuite_unittest.cpp @@ -0,0 +1,112 @@ +// +// Copyright 2019 The ANGLE Project Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +// TestSuite_unittest.cpp: Unit tests for ANGLE's test harness. +// + +#include + +#include "../angle_test_instantiate.h" +#include "TestSuite.h" +#include "common/debug.h" +#include "common/system_utils.h" +#include "util/test_utils.h" +#include "util/test_utils_unittest_helper.h" + +#include + +using namespace angle; + +namespace js = rapidjson; + +namespace +{ +constexpr char kTestHelperExecutable[] = "test_utils_unittest_helper"; + +class TestSuiteTest : public testing::Test +{ + protected: + void TearDown() override + { + if (!mTempFileName.empty()) + { + angle::DeleteFile(mTempFileName.c_str()); + } + } + + std::string mTempFileName; +}; + +// Tests the ANGLE standalone testing harness. Runs four tests with different ending conditions. +// Verifies that Pass, Fail, Crash and Timeout are all handled correctly. +TEST_F(TestSuiteTest, RunMockTests) +{ + std::string executablePath = GetExecutableDirectory(); + EXPECT_NE(executablePath, ""); + executablePath += std::string("/") + kTestHelperExecutable + GetExecutableExtension(); + + constexpr uint32_t kMaxTempDirLen = 100; + char tempFileName[kMaxTempDirLen * 2]; + ASSERT_TRUE(GetTempDir(tempFileName, kMaxTempDirLen)); + + std::stringstream tempFNameStream; + tempFNameStream << tempFileName << "/test_temp_" << rand() << ".json"; + mTempFileName = tempFNameStream.str(); + + std::string resultsFileName = "--results-file=" + mTempFileName; + + std::vector args = {executablePath.c_str(), + kRunTestSuite, + "--gtest_filter=MockTestSuiteTest.DISABLED_*", + "--gtest_also_run_disabled_tests", + "--bot-mode", + "--test-timeout=10", + resultsFileName.c_str()}; + + ProcessHandle process(args, true, true); + EXPECT_TRUE(process->started()); + EXPECT_TRUE(process->finish()); + EXPECT_TRUE(process->finished()); + EXPECT_EQ(process->getStderr(), ""); + + TestResults actual; + ASSERT_TRUE(GetTestResultsFromFile(mTempFileName.c_str(), &actual)); + EXPECT_TRUE(DeleteFile(mTempFileName.c_str())); + mTempFileName.clear(); + + std::map expectedResults = { + {{"MockTestSuiteTest", "DISABLED_Pass"}, {TestResultType::Pass, 0.0}}, + {{"MockTestSuiteTest", "DISABLED_Fail"}, {TestResultType::Fail, 0.0}}, + {{"MockTestSuiteTest", "DISABLED_Timeout"}, {TestResultType::Timeout, 0.0}}, + // {{"MockTestSuiteTest", "DISABLED_Crash"}, {TestResultType::Crash, 0.0}}, + }; + + EXPECT_EQ(expectedResults, actual.results); +} + +// Normal passing test. +TEST(MockTestSuiteTest, DISABLED_Pass) +{ + EXPECT_TRUE(true); +} + +// Normal failing test. +TEST(MockTestSuiteTest, DISABLED_Fail) +{ + EXPECT_TRUE(false); +} + +// Trigger a test timeout. +TEST(MockTestSuiteTest, DISABLED_Timeout) +{ + angle::Sleep(30000); +} + +// Trigger a test crash. +// TEST(MockTestSuiteTest, DISABLED_Crash) +// { +// ANGLE_CRASH(); +// } +} // namespace diff --git a/util/posix/crash_handler_posix.cpp b/util/posix/crash_handler_posix.cpp index 64c31a78ac897..a6d904a68daa0 100644 --- a/util/posix/crash_handler_posix.cpp +++ b/util/posix/crash_handler_posix.cpp @@ -35,7 +35,6 @@ namespace angle { - #if defined(ANGLE_PLATFORM_ANDROID) || defined(ANGLE_PLATFORM_FUCHSIA) void PrintStackBacktrace() @@ -54,6 +53,10 @@ void TerminateCrashHandler() } #else +namespace +{ +CrashCallback *gCrashHandlerCallback; +} // namespace # if defined(ANGLE_PLATFORM_APPLE) @@ -85,6 +88,11 @@ void PrintStackBacktrace() static void Handler(int sig) { + if (gCrashHandlerCallback) + { + (*gCrashHandlerCallback)(); + } + printf("\nSignal %d:\n", sig); PrintStackBacktrace(); @@ -127,6 +135,11 @@ void PrintStackBacktrace() static void Handler(int sig) { + if (gCrashHandlerCallback) + { + (*gCrashHandlerCallback)(); + } + printf("\nSignal %d [%s]:\n", sig, strsignal(sig)); PrintStackBacktrace(); @@ -142,6 +155,7 @@ static constexpr int kSignals[] = { void InitCrashHandler(CrashCallback *callback) { + gCrashHandlerCallback = callback; for (int sig : kSignals) { // Register our signal handler unless something's already done so (e.g. catchsegv). @@ -155,6 +169,7 @@ void InitCrashHandler(CrashCallback *callback) void TerminateCrashHandler() { + gCrashHandlerCallback = nullptr; for (int sig : kSignals) { void (*prev)(int) = signal(sig, SIG_DFL); diff --git a/util/test_utils_unittest_helper.cpp b/util/test_utils_unittest_helper.cpp index 2604f11c5f965..d8c478516f51c 100644 --- a/util/test_utils_unittest_helper.cpp +++ b/util/test_utils_unittest_helper.cpp @@ -7,12 +7,22 @@ #include "test_utils_unittest_helper.h" +#include "../src/tests/test_utils/runner/TestSuite.h" #include "common/system_utils.h" #include int main(int argc, char **argv) { + for (int argIndex = 1; argIndex < argc; ++argIndex) + { + if (strcmp(argv[argIndex], kRunTestSuite) == 0) + { + angle::TestSuite testSuite(&argc, argv); + return testSuite.run(); + } + } + if (argc != 3 || strcmp(argv[1], kRunAppTestArg1) != 0 || strcmp(argv[2], kRunAppTestArg2) != 0) { fprintf(stderr, "Expected command line:\n%s %s %s\n", argv[0], kRunAppTestArg1, diff --git a/util/test_utils_unittest_helper.h b/util/test_utils_unittest_helper.h index 660bde1afe793..6414e476b1d7f 100644 --- a/util/test_utils_unittest_helper.h +++ b/util/test_utils_unittest_helper.h @@ -16,6 +16,7 @@ constexpr char kRunAppTestStdout[] = "RunAppTest stdout test\n"; constexpr char kRunAppTestStderr[] = "RunAppTest stderr test\n .. that expands multiple lines\n"; constexpr char kRunAppTestArg1[] = "--expected-arg1"; constexpr char kRunAppTestArg2[] = "expected_arg2"; +constexpr char kRunTestSuite[] = "--run-test-suite"; } // anonymous namespace #endif // COMMON_SYSTEM_UTILS_UNITTEST_HELPER_H_