feat(test): run tests and update build result (#88)

JohelEGP · web-flow · commit 472e744828eb · 2023-08-22T11:04:21.000-04:00
* refactor(test): update results as fixture cleanups This is less wasteful as it makes use of the output of existing tests. It should simplify adding support for running the tests. * refactor(test): order build after codegen Ordering by relative cost reduces turnarounds. Previously, codegen update happened last. * feat(test): run tests and update build result Although the project's language is now `CXX`, the setup of the C++ environment is as was before. I.e., - The environment of the build test can only be set globally (through CMake defaults, the environment, etc.). - The environment of this project should match that of the build test. It doesn't attempt to propagate any aspect explicitly. * fix(test): remove outdated condition See <https://cpp2.godbolt.org/z/4r1P6f34E>. * refactor: execute tests without developer mode * feat(test): copy new or changed build results only * feat(test): check expected build results * fix(test): add non-`check` tests when developing Test results will be updated, so make the tests use those results without a reconfiguration in-between. Also, trigger reconfiguration to add the `check` tests. * fix(test): add glob for new error tests * fix: account for C++23 tests
diff --git a/README.md b/README.md
@@ -139,4 +139,4 @@ processing the directory.
 
 The CMake project `regression-tests/CMakeLists.txt` runs the test suite of cppfront.
 See "Regression tests" at [`./.github/workflows/ci.yml`](./.github/workflows/ci.yml) for how to set it up.
-To update the test results, build with `--target cppfront_update_test_results`.
+To have the test results updated, configure with `-DCPPFRONT_DEVELOPING=TRUE`.
diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt
@@ -1,15 +1,61 @@
 cmake_minimum_required(VERSION 3.23)
-project(cppfront-regression-tests LANGUAGES NONE)
+project(cppfront-regression-tests LANGUAGES CXX)
+
+option(CPPFRONT_DEVELOPING "Test results are updated.")
 
 enable_testing()
 
 set(CPPFRONT_NO_MAGIC 1)
 find_package(cppfront REQUIRED)
 
 set(REGRESSION_TESTS_DIR "${CMAKE_CURRENT_LIST_DIR}/../cppfront/regression-tests")
+set(TEST_RESULTS_DIR "${REGRESSION_TESTS_DIR}/test-results")
+
+# Set `COMPILER_ITEM_NAME`.
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set(compiler_id "gcc")
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
+    set(compiler_id "apple-clang")
+else ()
+    string(TOLOWER "${CMAKE_CXX_COMPILER_ID}" compiler_id)
+endif ()
+string(REGEX MATCH "[0-9]+" compiler_major_version "${CMAKE_CXX_COMPILER_VERSION}")
+set(COMPILER_ITEM_NAME "${compiler_id}-${compiler_major_version}")
+
+# Setup `BUILD_RESULTS_DIR`.
+set(BUILD_RESULTS_DIR "${TEST_RESULTS_DIR}/${COMPILER_ITEM_NAME}")
+
+if (CPPFRONT_DEVELOPING)
+    file(MAKE_DIRECTORY "${BUILD_RESULTS_DIR}")
+
+    # Write compiler version output.
+    set(compiler_version_command "${CMAKE_CXX_COMPILER}" "--version")
+    if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
+        set(compiler_id "clang")
+    endif ()
+    execute_process(
+        COMMAND ${compiler_version_command}
+        OUTPUT_FILE "${BUILD_RESULTS_DIR}/${compiler_id}-version.output"
+    )
+endif ()
+
+configure_file("cmake/ExecuteWithRedirection.cmake" "ExecuteWithRedirection.cmake" COPYONLY)
+configure_file("cmake/ExecuteTestCase.cmake.in" "ExecuteTestCase.cmake" @ONLY)
+configure_file("cmake/FindBuildResultFile.cmake.in" "FindBuildResultFile.cmake" @ONLY)
+configure_file("cmake/UpdateBuildOutput.cmake.in" "UpdateBuildOutput.cmake" @ONLY)
 
-configure_file("cmake/UpdateTestResults.cmake.in" "UpdateTestResults.cmake" @ONLY)
-add_custom_target(cppfront_update_test_results)
+include("${CMAKE_CURRENT_BINARY_DIR}/FindBuildResultFile.cmake")
+
+function(cppfront_add_check_test)
+    cmake_parse_arguments(PARSE_ARGV 0 ARG "" "NAME;NEW_FILE;OLD_FILE;FIXTURES_REQUIRED" "")
+
+    add_test(
+        NAME "${ARG_NAME}"
+        COMMAND "${CMAKE_COMMAND}" -E compare_files "${ARG_NEW_FILE}" "${ARG_OLD_FILE}"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+    )
+    set_tests_properties("${ARG_NAME}" PROPERTIES FIXTURES_REQUIRED "${ARG_FIXTURES_REQUIRED}")
+endfunction()
 
 function(cppfront_command_tests)
     cmake_parse_arguments(PARSE_ARGV 0 ARG "" "SOURCE;EXPECTED_FILE" "EXTRA_FLAGS")
@@ -18,24 +64,39 @@ function(cppfront_command_tests)
 
     cmake_path(GET ARG_SOURCE STEM test_name)
 
-    file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${test_name}")
-    add_custom_target(
-        "${test_name}"
-        COMMAND
-        "${CMAKE_COMMAND}"
-        -D "TEST_NAME=${test_name}"
-        -D "EXTRA_FLAGS=${ARG_EXTRA_FLAGS}"
-        -P "UpdateTestResults.cmake"
-    )
-    add_dependencies(cppfront_update_test_results "${test_name}")
-
+    if (NOT "${test_name}" MATCHES [[.*-error$]])
+        set(gen_cpp_src "${test_name}.cpp")
+        set(COMMAND_ERROR_IS_FATAL "COMMAND_ERROR_IS_FATAL" "ANY")
+    endif ()
     add_test(
         NAME "codegen/${test_name}"
-        COMMAND cppfront::cppfront "${ARG_SOURCE}" ${ARG_EXTRA_FLAGS}
+        COMMAND
+        "${CMAKE_COMMAND}"
+        -D "OUTPUT_FILE=${ARG_SOURCE}.output"
+        -P "ExecuteWithRedirection.cmake"
+        --
+        "${CPPFRONT_EXECUTABLE}" "${ARG_SOURCE}" ${ARG_EXTRA_FLAGS}
+        ${COMMAND_ERROR_IS_FATAL}
         WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
     )
+    set_tests_properties("codegen/${test_name}" PROPERTIES FIXTURES_SETUP "codegen/${test_name}")
 
-    set(expected_output_file "${REGRESSION_TESTS_DIR}/test-results/${ARG_SOURCE}.output")
+    unset(build_test_depends)
+    if (CPPFRONT_DEVELOPING)
+        add_test(
+            NAME "codegen/update/${test_name}"
+            COMMAND
+            "${CMAKE_COMMAND}"
+            -E copy_if_different
+            "${gen_cpp_src}"
+            "${ARG_SOURCE}.output"
+            "${TEST_RESULTS_DIR}"
+        )
+        set_tests_properties("codegen/update/${test_name}" PROPERTIES FIXTURES_CLEANUP "codegen/${test_name}")
+        set(build_test_depends "codegen/update/${test_name}")
+    endif ()
+
+    set(expected_output_file "${TEST_RESULTS_DIR}/${ARG_SOURCE}.output")
     if (EXISTS "${expected_output_file}")
         file(READ "${expected_output_file}" expected_output)
         string(REPLACE "\\" "\\\\" expected_output "${expected_output}")
@@ -48,25 +109,24 @@ function(cppfront_command_tests)
         string(REPLACE "*" "\\*" expected_output "${expected_output}")
         string(REPLACE "?" "\\?" expected_output "${expected_output}")
 
-        set_tests_properties(
-            "codegen/${test_name}"
-            PROPERTIES
-            FIXTURES_SETUP "codegen/${test_name}"
-            PASS_REGULAR_EXPRESSION "^${expected_output}$"
-        )
+        set_tests_properties("codegen/${test_name}" PROPERTIES PASS_REGULAR_EXPRESSION "^${expected_output}$")
     endif ()
 
-    if (ARG_EXPECTED_FILE)
-        cmake_path(REPLACE_EXTENSION ARG_SOURCE "cpp" OUTPUT_VARIABLE gen_cpp_src)
-
-        add_test(
+    if (EXISTS "${ARG_EXPECTED_FILE}")
+        configure_file("${ARG_EXPECTED_FILE}" "${gen_cpp_src}.original" COPYONLY)
+        cppfront_add_check_test(
             NAME "codegen/check/${test_name}"
-            COMMAND "${CMAKE_COMMAND}" -E compare_files "${gen_cpp_src}" "${ARG_EXPECTED_FILE}"
-            WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+            NEW_FILE "${gen_cpp_src}"
+            OLD_FILE "${gen_cpp_src}.original"
+            FIXTURES_REQUIRED "codegen/${test_name}"
         )
+        list(APPEND build_test_depends "codegen/check/${test_name}")
+    elseif (CPPFRONT_DEVELOPING)
+        # Trigger regeneration to add the `check` tests for newly added results.
+        file(GLOB unused CONFIGURE_DEPENDS "${ARG_EXPECTED_FILE}")
+    endif ()
 
-        set_tests_properties("codegen/check/${test_name}" PROPERTIES FIXTURES_REQUIRED "codegen/${test_name}")
-
+    if (DEFINED build_test_depends AND DEFINED gen_cpp_src)
         cppfront_build_tests(
             SOURCE ${ARG_SOURCE}
             EXTRA_FLAGS ${ARG_EXTRA_FLAGS}
@@ -77,26 +137,25 @@ endfunction()
 function(cppfront_build_tests)
     cmake_parse_arguments(PARSE_ARGV 0 ARG "" "SOURCE" "EXTRA_FLAGS")
 
-    # The following tests aren't expected to be buildable C++, even if
-    # cppfront succeeds.
-    set(
-        codegen_only_tests
-        mixed-postfix-expression-custom-formatting
-    )
-
-    cmake_path(GET ARG_SOURCE STEM stem)
-    if (stem IN_LIST codegen_only_tests)
-        return()
-    endif ()
+    cmake_path(GET ARG_SOURCE STEM test_name)
 
-    set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/${stem}")
-    set(test_name "build/${stem}")
+    set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/${test_name}")
 
     configure_file("${REGRESSION_TESTS_DIR}/${ARG_SOURCE}" "${test_dir}/${ARG_SOURCE}" COPYONLY)
     configure_file("cmake/CMakeLists.txt.in" "${test_dir}/CMakeLists.txt" @ONLY)
+    configure_file("cmake/test-case-config.cmake.in" "${test_dir}/test-case-config.cmake.in" COPYONLY)
+
+    set(
+        cxx_23_tests
+        pure2-bugfix-for-empty-index
+    )
+    set(extra_flags)
+    if (test_name IN_LIST cxx_23_tests)
+        list(APPEND extra_flags "-DCMAKE_CXX_STANDARD=23")
+    endif ()
 
     add_test(
-        NAME "${test_name}"
+        NAME "build/${test_name}"
         COMMAND
         "${CMAKE_CTEST_COMMAND}"
         --build-and-test "${test_dir}" "${test_dir}/build"
@@ -107,7 +166,74 @@ function(cppfront_build_tests)
         "-Dcppfront_DIR=${cppfront_DIR}"
         "-Dcppfront-exe_DIR=${cppfront-exe_DIR}"
         "-DCPPFRONT_FLAGS=${ARG_EXTRA_FLAGS}"
+        ${extra_flags}
+        # There's `CMAKE_CXX_LINKER_LAUNCHER`, too. So far, it's not needed.
+        "-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_COMMAND};-D;OUTPUT_FILE=${gen_cpp_src}.output;-P;../../ExecuteWithRedirection.cmake;--"
     )
+    set_tests_properties(
+        "build/${test_name}"
+        PROPERTIES
+        DEPENDS "${build_test_depends}"
+        FIXTURES_SETUP "build/${test_name}"
+    )
+
+    cppfront_find_build_result_file(expected_output_file RESULT_FILE "${gen_cpp_src}.output")
+    if (expected_output_file)
+        cppfront_add_check_test(
+            NAME "build/check/${test_name}"
+            NEW_FILE "${test_dir}/build/${gen_cpp_src}.output"
+            OLD_FILE "${expected_output_file}"
+            FIXTURES_REQUIRED "build/${test_name}"
+        )
+    endif()
+
+    add_test(
+        NAME "build/execute/${test_name}"
+        COMMAND
+        "${CMAKE_COMMAND}"
+        -D "OUTPUT_FILE=${test_dir}/${gen_cpp_src}.execution"
+        -P "../ExecuteTestCase.cmake"
+        WORKING_DIRECTORY "${test_dir}"
+    )
+    set_tests_properties(
+        "build/execute/${test_name}"
+        PROPERTIES
+        FIXTURES_REQUIRED "build/${test_name}"
+        FIXTURES_SETUP "build/execute/${test_name}"
+        RESOURCE_LOCK "test.exe")
+
+    cppfront_find_build_result_file(expected_execution_file RESULT_FILE "${gen_cpp_src}.execution")
+    if (expected_execution_file)
+        cppfront_add_check_test(
+            NAME "build/execute/check/${test_name}"
+            NEW_FILE "${test_dir}/${gen_cpp_src}.execution"
+            OLD_FILE "${expected_execution_file}"
+            FIXTURES_REQUIRED "build/execute/${test_name}"
+        )
+    endif()
+
+    if (CPPFRONT_DEVELOPING)
+        # Trigger regeneration to add the `check` tests for newly added results.
+        if (NOT DEFINED expected_output_file OR NOT DEFINED expected_execution_file)
+            file(GLOB unused CONFIGURE_DEPENDS "${BUILD_RESULTS_DIR}/${gen_cpp_src}.*")
+        endif()
+
+        add_test(
+            NAME "build/update/${test_name}"
+            COMMAND
+            "${CMAKE_COMMAND}"
+            -D "GEN_CPP_SRC=${gen_cpp_src}"
+            -D "OUTPUT_FILE=build/${gen_cpp_src}.output"
+            -D "EXECUTION_FILE=${gen_cpp_src}.execution"
+            -P "../UpdateBuildOutput.cmake"
+            WORKING_DIRECTORY "${test_dir}"
+        )
+        set_tests_properties(
+            "build/update/${test_name}"
+            PROPERTIES
+            FIXTURES_CLEANUP "build/${test_name};build/execute/${test_name}"
+        )
+    endif ()
 endfunction()
 
 function(cppfront_tests)
@@ -119,14 +245,15 @@ function(cppfront_tests)
         RELATIVE "${REGRESSION_TESTS_DIR}"
         CONFIGURE_DEPENDS "${REGRESSION_TESTS_DIR}/${ARG_GROUP}-*.cpp2"
     )
+    # Trigger regeneration to recognize as a test to fail codegen.
+    file(
+        GLOB unused
+        CONFIGURE_DEPENDS "${REGRESSION_TESTS_DIR}/${ARG_GROUP}-*-error.cpp2"
+    )
 
     foreach (src IN LISTS sources)
         cmake_path(REPLACE_EXTENSION src "cpp" OUTPUT_VARIABLE expected_file)
-        set(expected_file "${REGRESSION_TESTS_DIR}/test-results/${expected_file}")
-
-        if (NOT EXISTS "${expected_file}")
-            set(expected_file "")
-        endif ()
+        set(expected_file "${TEST_RESULTS_DIR}/${expected_file}")
 
         cppfront_command_tests(
             SOURCE ${src}
diff --git a/regression-tests/cmake/CMakeLists.txt.in b/regression-tests/cmake/CMakeLists.txt.in
@@ -13,3 +13,30 @@ target_compile_options(
     PRIVATE
     "$<$<CXX_COMPILER_ID:MSVC>:/experimental:module>"
 )
+
+include(GNUInstallDirs)
+include(CMakePackageConfigHelpers)
+
+set(TEST_CASE_INSTALL_CMAKEDIR "${CMAKE_INSTALL_DATADIR}/cmake/test-case")
+
+install(
+    TARGETS test-case
+    EXPORT test-case-targets
+)
+
+install(
+    EXPORT test-case-targets
+    DESTINATION "${TEST_CASE_INSTALL_CMAKEDIR}"
+)
+
+configure_package_config_file(
+    "test-case-config.cmake.in"
+    "test-case-config.cmake"
+    INSTALL_DESTINATION "${TEST_CASE_INSTALL_CMAKEDIR}"
+    PATH_VARS CMAKE_INSTALL_BINDIR
+)
+
+install(
+    FILES "${CMAKE_CURRENT_BINARY_DIR}/test-case-config.cmake"
+    DESTINATION "${TEST_CASE_INSTALL_CMAKEDIR}"
+)
diff --git a/regression-tests/cmake/ExecuteTestCase.cmake.in b/regression-tests/cmake/ExecuteTestCase.cmake.in
@@ -0,0 +1,23 @@
+execute_process(
+    COMMAND "${CMAKE_COMMAND}" --install "build" --prefix "_local"
+    COMMAND_ERROR_IS_FATAL ANY
+)
+find_package(test-case REQUIRED PATHS "${CMAKE_CURRENT_BINARY_DIR}/_local" NO_DEFAULT_PATH)
+execute_process(
+    COMMAND
+    "${CMAKE_COMMAND}"
+    -E copy
+    "${TEST_CASE_EXECUTABLE}"
+    "@BUILD_RESULTS_DIR@/test.exe"
+)
+execute_process(
+    COMMAND
+    "${CMAKE_COMMAND}"
+    -D "OUTPUT_FILE=${OUTPUT_FILE}"
+    -P "@CMAKE_CURRENT_BINARY_DIR@/ExecuteWithRedirection.cmake"
+    --
+    "./test.exe"
+    COMMAND_ERROR_IS_FATAL ANY
+    WORKING_DIRECTORY "@BUILD_RESULTS_DIR@"
+)
+execute_process(COMMAND "${CMAKE_COMMAND}" -E rm "@BUILD_RESULTS_DIR@/test.exe")
diff --git a/regression-tests/cmake/ExecuteWithRedirection.cmake b/regression-tests/cmake/ExecuteWithRedirection.cmake
@@ -0,0 +1,14 @@
+if (NOT "${CMAKE_ARGV5}" STREQUAL "--")
+    message(FATAL_ERROR "Unexpected argument.")
+endif ()
+foreach (i RANGE 6 ${CMAKE_ARGC})
+    list(APPEND command_args "${CMAKE_ARGV${i}}") # Probably doesn't handle nested `;`.
+endforeach ()
+execute_process(
+    COMMAND ${command_args}
+    OUTPUT_VARIABLE OUTPUT
+    ERROR_VARIABLE OUTPUT
+    ECHO_OUTPUT_VARIABLE
+    ECHO_ERROR_VARIABLE
+)
+file(WRITE "${OUTPUT_FILE}" "${OUTPUT}")
diff --git a/regression-tests/cmake/FindBuildResultFile.cmake.in b/regression-tests/cmake/FindBuildResultFile.cmake.in
@@ -0,0 +1,28 @@
+# Set `OLD_BUILD_RESULTS_DIRS`.
+file(
+    GLOB build_result_dirs
+    RELATIVE "@TEST_RESULTS_DIR@"
+    "@TEST_RESULTS_DIR@/@compiler_id@-*"
+)
+list(SORT build_result_dirs)
+list(FIND build_result_dirs "@COMPILER_ITEM_NAME@" i)
+list(SUBLIST build_result_dirs 0 ${i} OLD_BUILD_RESULTS_DIRS)
+list(REVERSE OLD_BUILD_RESULTS_DIRS)
+
+function(cppfront_find_build_result_file out_var)
+    cmake_parse_arguments(PARSE_ARGV 0 ARG "OLD_ONLY" "RESULT_FILE" "")
+
+    if (NOT ARG_OLD_ONLY)
+        set(extra_item "@COMPILER_ITEM_NAME@")
+    endif ()
+
+    foreach (build_results_dir IN ITEMS "${extra_item}" LISTS OLD_BUILD_RESULTS_DIRS)
+        set(result_file "@TEST_RESULTS_DIR@/${build_results_dir}/${ARG_RESULT_FILE}")
+        if (EXISTS "${result_file}")
+            set("${out_var}" "${result_file}" PARENT_SCOPE)
+            return()
+        endif ()
+    endforeach ()
+
+    unset("${out_var}" PARENT_SCOPE)
+endfunction()
diff --git a/regression-tests/cmake/UpdateBuildOutput.cmake.in b/regression-tests/cmake/UpdateBuildOutput.cmake.in
diff --git a/regression-tests/cmake/UpdateTestResults.cmake.in b/regression-tests/cmake/UpdateTestResults.cmake.in
diff --git a/regression-tests/cmake/test-case-config.cmake.in b/regression-tests/cmake/test-case-config.cmake.in
