From befa6656b8aa11065659193902eeb663c0944b32 Mon Sep 17 00:00:00 2001 From: Joseph Hickey Date: Wed, 26 Feb 2020 15:38:27 -0500 Subject: [PATCH] Fix #362, integrate coverage with UT build Build the full suite of OSAL coverage tests as part of the normal build when ENABLE_UNIT_TESTS is true. This will use the same target OSAL BSP to execute the tests as is used for the FSW. --- CMakeLists.txt | 6 + src/unit-test-coverage/CMakeLists.txt | 127 +++++------------- .../posix/modules/CMakeLists.txt | 20 ++- .../ut-stubs/src/osapi-impl-posix-stubs.c | 1 - .../shared/modules/CMakeLists.txt | 10 +- .../ut-stubs/CMakeLists.txt | 13 +- src/unit-test-coverage/vxworks/CMakeLists.txt | 3 - .../vxworks/modules/CMakeLists.txt | 10 +- src/unit-tests/CMakeLists.txt | 16 +-- 9 files changed, 77 insertions(+), 129 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 69eda30b5..e0852e5dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -241,6 +241,12 @@ if (ENABLE_UNIT_TESTS) # testing other application code built on top of OSAL. add_subdirectory(src/ut-stubs ut-stubs) + # The "unit-test-coverage" is a white-box line coverage test. + # It re-compiles each source unit for coverage analysis and links + # with stub dependencies and a test sequence designed to execute + # every line of source code within OSAL. + add_subdirectory(src/unit-test-coverage unit-test-coverage) + # The "tests" and "unit-tests" subdirectories implement examples and verification tests # of the OSAL library. Note that these are both black box tests that link with the full # OSAL (not a stub/coverage test). diff --git a/src/unit-test-coverage/CMakeLists.txt b/src/unit-test-coverage/CMakeLists.txt index b4128112a..329d050bf 100644 --- a/src/unit-test-coverage/CMakeLists.txt +++ b/src/unit-test-coverage/CMakeLists.txt @@ -5,15 +5,15 @@ # To run coverage tests this must be built separately. There are several reasons # for having it this way: # - Not all targets have coverage testing implemented (yet). Only VxWorks right now. -# - It may use an entirely different toolchain than the actual target build. +# - It may use an entirely different toolchain than the actual target build. # (coverage tests run with the "real" calls stubbed out, so they can be executed # on any platform/os - case in point, the VxWorks coveraged paths can be analyzed -# by running the code on Linux using the specially-crafted inputs) -# - By definition this MUST completely rebuild OSAL to add the requisite "coverage" +# by running the code on Linux using the specially-crafted inputs) +# - By definition this MUST completely rebuild OSAL to add the requisite "coverage" # options so that the binaries include the extra instrumentation. # -# NO ARTEFACTS FROM THIS BUILD SHOULD EVER INTERMINGLE WITH THE REAL TARGET BUILD -# +# NO ARTEFACTS FROM THIS BUILD SHOULD EVER INTERMINGLE WITH THE REAL TARGET BUILD +# cmake_minimum_required(VERSION 2.6.4) project(OSALCOVERAGE C) @@ -21,36 +21,16 @@ project(OSALCOVERAGE C) # Ask to generate a "make test" target enable_testing() -add_definitions(-Wall -Werror) -add_definitions(-D_UNIT_TEST_) - -# Assume that this script lives in a subdirectory called /src/unit-test-coverage -# Get a reference to the top-level OSAL source tree and store it in OSAL_SOURCE_DIR -get_filename_component(OSAL_SOURCE_DIR "${CMAKE_SOURCE_DIR}/../.." ABSOLUTE) - - -# Cache any user-specified C flags so they will be retained in future builds -# These can be specified either through cmake command line (e.g. -DUSER_C_FLAGS=-Werror) or -# through an environment variable (e.g. OSALCOVERAGE_USER_C_FLAGS=-Werror cmake ...) -set(OSALCOVERAGE_USER_C_FLAGS "$ENV{OSALCOVERAGE_USER_C_FLAGS}" CACHE STRING "User-specified C flags for OSAL coverage test build") - # The following logic is borrowed from the real OSAL build # One difference here is that the UT BSP/framework and target OSTYPE need not "match" # The following cache variables are recognized: # OSALCOVERAGE_TARGET_OSTYPE -> the intended OSAL that runs on the actual target -# OSALCOVERAGE_HOST_BSPTYPE -> the platform/bsp that will execute the coverage tests -# -# The currently supported setup is to use the "pc-linux" BSP to execute the "vxworks6" +# +# The currently supported setup is to use the configured BSP to execute all available # code coverage analysis. Because the actual underlying OS calls are stubbed out, there -# is no dependency on the actual underlying OS. -set(OSALCOVERAGE_TARGET_OSTYPE "vxworks;shared" CACHE STRING "OSAL target(s) to build coverage tests for") -set(OSALCOVERAGE_HOST_BSPTYPE "pc-linux" CACHE STRING "OSAL unit test BSP to execute coverage tests") - -# OSALCOVERAGE_SYSTEM_OSTYPE indicates which of the BSP packages to include -# This is required and must be defined -if (NOT OSALCOVERAGE_TARGET_OSTYPE) - message(FATAL_ERROR "OSALCOVERAGE_TARGET_OSTYPE must be set to the appropriate OS") -endif (NOT OSALCOVERAGE_TARGET_OSTYPE) +# is no dependency on the actual underlying OS. Note that RTEMS is not included as the +# coverage test is not implemented at this time. +set(OSALCOVERAGE_TARGET_OSTYPE "vxworks;posix;shared" CACHE STRING "OSAL target(s) to build coverage tests for (default=all)") # Check that coverage has been implemented for this OSTYPE foreach(OSTYPE ${OSALCOVERAGE_TARGET_OSTYPE}) @@ -59,60 +39,12 @@ foreach(OSTYPE ${OSALCOVERAGE_TARGET_OSTYPE}) endif (NOT IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${OSTYPE}) endforeach(OSTYPE ${OSALCOVERAGE_TARGET_OSTYPE}) -# Do NOT Include the OS-specific compiler options (those are for normal builds) -#include(src/os/${OSALCOVERAGE_TARGET_OSTYPE}/build_options.cmake OPTIONAL) message(STATUS "Coverage Test Target OS: ${OSALCOVERAGE_TARGET_OSTYPE}") -# OSALCOVERAGE_SYSTEM_BSPTYPE indicates which of the BSP packages to use to execute the tests -if (NOT OSALCOVERAGE_HOST_BSPTYPE) - message(FATAL_ERROR "OSALCOVERAGE_HOST_BSPTYPE must be set to the appropriate BSP") -endif (NOT OSALCOVERAGE_HOST_BSPTYPE) - -if (NOT IS_DIRECTORY ${OSAL_SOURCE_DIR}/src/bsp/${OSALCOVERAGE_HOST_BSPTYPE}) - message(FATAL_ERROR "${OSALCOVERAGE_HOST_BSPTYPE} is not a valid host BSP") -endif (NOT IS_DIRECTORY ${OSAL_SOURCE_DIR}/src/bsp/${OSALCOVERAGE_HOST_BSPTYPE}) - -# Include the BSP-specific (host) compiler options -include(${OSAL_SOURCE_DIR}/src/bsp/${OSALCOVERAGE_HOST_BSPTYPE}/make/build_options.cmake OPTIONAL) -message(STATUS "Coverage Test Host BSP: ${OSALCOVERAGE_HOST_BSPTYPE}") - -set(CMAKE_C_FLAGS "${OSALCOVERAGE_C_FLAGS} ${OSAL_C_FLAGS}") -message(STATUS "Coverage Test CFLAGS: ${CMAKE_C_FLAGS}") - # Utilize the shared UT assert library, along with the standard OSAL includes -include_directories(${OSAL_SOURCE_DIR}/ut_assert/inc) +include_directories(${UT_ASSERT_SOURCE_DIR}/inc) include_directories(${OSAL_SOURCE_DIR}/src/os/inc) - -# The "osconfig.h" file for coverage test should ideally be the same one that -# is used for the normal build. The path to this file can be supplied as OSAL_INCLUDEDIR -# If not supplied, the file from the UT BSP can be used, but coverage results might be -# less accurate. -if (DEFINED OSAL_INCLUDEDIR) - include_directories(${OSAL_INCLUDEDIR}) -else (DEFINED OSAL_INCLUDEDIR) - message(STATUS "No OSAL_INCLUDEDIR specified, using default config from UT BSP") - include_directories(${OSAL_SOURCE_DIR}/src/bsp/${OSALCOVERAGE_HOST_BSPTYPE}/config) -endif (DEFINED OSAL_INCLUDEDIR) - -# These include paths are not public APIs but are needed -# to get to some internal headers for private APIs include_directories(${OSAL_SOURCE_DIR}/src/os/shared) -include_directories(${OSAL_SOURCE_DIR}/src/os/portable) -include_directories(${CMAKE_SOURCE_DIR}/ut-stubs/inc) - -# The "ut_bsp" library is a simple startup BSP that can be used for unit testing -# This removes the need to use the "real" CFE PSP and also provides the necessary -# UT output functions that UT assert may rely upon to report test messages -# This is the OSAL BSP but with the _UNIT_TEST_ macro defined so it may have UT-specific features -aux_source_directory(${OSAL_SOURCE_DIR}/src/bsp/${OSALCOVERAGE_HOST_BSPTYPE}/src BSPFILES) -add_library(ut_bsp STATIC ${BSPFILES}) - -# The "utassert" library is the core GSFC-provided unit test library -# It is only the generic framework and does _not_ include any of the specific stub/hook functions -# It is built as static library so it may be linked with either a "real" implementation or a stub -# library (see next targets) or some combination of those as the test cases dictate. -aux_source_directory(${OSAL_SOURCE_DIR}/ut_assert/src UT_ASSERT_FILES) -add_library(ut_assert STATIC ${UT_ASSERT_FILES}) # The OSALCOVERAGE_STUB_LIB_LIST is a list of stub libraries to link the # test runner executables with. It will be appended at various points @@ -120,12 +52,8 @@ add_library(ut_assert STATIC ${UT_ASSERT_FILES}) set(OSALCOVERAGE_STUB_LIB_LIST) add_subdirectory(ut-stubs) -# Build the stub libraries for OSAL itself. -# These are potentially needed by many tests, to handle calls -# from the module under test into OTHER parts of OSAL -aux_source_directory(${OSAL_SOURCE_DIR}/src/ut-stubs UT_OSAPI_STUB_FILES) -add_library(ut_osapi_stubs STATIC ${UT_OSAPI_STUB_FILES}) - +# The "ut_osapi_stubs" library is the stubs for the OSAL public API +# this should be supplied by the parent build list(APPEND OSALCOVERAGE_STUB_LIB_LIST ut_osapi_stubs) # A generic function to add a coverage test case source file @@ -136,7 +64,7 @@ list(APPEND OSALCOVERAGE_STUB_LIB_LIST ut_osapi_stubs) function (add_coverage_tests SETNAME) foreach(MODNAME ${ARGN}) set (TESTCASE_SRCFILE) - foreach (SRCFILE + foreach (SRCFILE "${PROJECT_SOURCE_DIR}/portable/coveragetest-${MODNAME}.c" "${CMAKE_CURRENT_SOURCE_DIR}/src/coveragetest-${MODNAME}.c" ) @@ -144,27 +72,34 @@ function (add_coverage_tests SETNAME) set (TESTCASE_SRCFILE "${SRCFILE}") endif (EXISTS "${SRCFILE}") endforeach() - + if (TESTCASE_SRCFILE) - set(TESTNAME "${SETNAME}-${MODNAME}") - message (STATUS "Found test case for ${TESTNAME} in ${TESTCASE_SRCFILE}") - + set(TESTNAME "coverage-${SETNAME}-${MODNAME}") + if (DEFINED MODULE_LINK_MAP_${MODNAME}) set(LINKMOD ${MODULE_LINK_MAP_${MODNAME}}) else() set(LINKMOD ${MODNAME}) endif() - - add_executable(${TESTNAME}-testrunner + + add_executable(${TESTNAME}-testrunner ${TESTCASE_SRCFILE} $) - - set_target_properties(${TESTNAME}-testrunner PROPERTIES LINK_FLAGS "${UT_C_FLAGS}") - target_link_libraries(${TESTNAME}-testrunner ${OSALCOVERAGE_STUB_LIB_LIST} ut_bsp ut_assert) + + target_link_libraries(${TESTNAME}-testrunner + ${UT_COVERAGE_LINK_FLAGS} + ${OSALCOVERAGE_STUB_LIB_LIST} + ut_assert + ) add_test(${TESTNAME} ${TESTNAME}-testrunner) + + foreach(TGT ${INSTALL_TARGET_LIST}) + install(TARGETS ${TESTNAME}-testrunner DESTINATION ${TGTNAME}/${UT_INSTALL_SUBDIR}) + endforeach() + endif() endforeach(MODNAME ${ARGN}) - + endfunction(add_coverage_tests SETNAME) diff --git a/src/unit-test-coverage/posix/modules/CMakeLists.txt b/src/unit-test-coverage/posix/modules/CMakeLists.txt index c48a887bf..98fdcb0d3 100644 --- a/src/unit-test-coverage/posix/modules/CMakeLists.txt +++ b/src/unit-test-coverage/posix/modules/CMakeLists.txt @@ -17,19 +17,25 @@ # - Each sub-module is compiled to a separate "OBJECT" library # this simplifies the testrunner linking by allow only one specific # object file to be linked into each test -# - Each sub-module is wrapped to divert C library calls to the -# stub (OCS) counterpart -# - UT_C_FLAGS are enabled to include any code coverage instrumentation +# - Each sub-module is wrapped to divert C library calls to the +# stub (OCS) counterpart +# - UT_COVERAGE_COMPILE_FLAGS are enabled to include any code coverage instrumentation # the "overrides" dir contains empty versions of the C-library include files. include_directories(${OSALCOVERAGE_SOURCE_DIR}/ut-stubs/inc) include_directories(${OSALCOVERAGE_SOURCE_DIR}/ut-stubs/override_inc) +include_directories(${OSAL_SOURCE_DIR}/src/bsp/shared) +include_directories(${OSAL_SOURCE_DIR}/src/os/shared) foreach(MODULE ${MODULE_LIST}) if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/src/ut-${MODULE}.c) - add_library(ut_${SETNAME}_${MODULE} OBJECT src/ut-${MODULE}.c) - set_target_properties(ut_${SETNAME}_${MODULE} PROPERTIES COMPILE_FLAGS "${UT_C_FLAGS}") + add_library(ut_${SETNAME}_${MODULE} OBJECT + src/ut-${MODULE}.c + ) + target_compile_options(ut_${SETNAME}_${MODULE} PRIVATE + ${UT_COVERAGE_COMPILE_FLAGS} + ) endif () endforeach() - - + + diff --git a/src/unit-test-coverage/posix/ut-stubs/src/osapi-impl-posix-stubs.c b/src/unit-test-coverage/posix/ut-stubs/src/osapi-impl-posix-stubs.c index d519a8894..f1dc87f7c 100644 --- a/src/unit-test-coverage/posix/ut-stubs/src/osapi-impl-posix-stubs.c +++ b/src/unit-test-coverage/posix/ut-stubs/src/osapi-impl-posix-stubs.c @@ -1,7 +1,6 @@ #include #include #include "utstubs.h" -#include "os-posix.h" UT_DEFAULT_STUB(OS_Posix_TimeBaseAPI_Impl_Init, (void)) UT_DEFAULT_STUB(OS_Posix_ModuleAPI_Impl_Init, (void)) diff --git a/src/unit-test-coverage/shared/modules/CMakeLists.txt b/src/unit-test-coverage/shared/modules/CMakeLists.txt index 0b35829ef..4cd649bbd 100644 --- a/src/unit-test-coverage/shared/modules/CMakeLists.txt +++ b/src/unit-test-coverage/shared/modules/CMakeLists.txt @@ -19,7 +19,7 @@ # object file to be linked into each test # - Each sub-module is wrapped to divert C library calls to the # stub (OCS) counterpart -# - UT_C_FLAGS are enabled to include any code coverage instrumentation +# - UT_COVERAGE_COMPILE_FLAGS are enabled to include any code coverage instrumentation # the "override_inc" dir contains replacement versions of the C-library include files. include_directories(${OSALCOVERAGE_SOURCE_DIR}/ut-stubs/inc) @@ -27,8 +27,12 @@ include_directories(${OSALCOVERAGE_SOURCE_DIR}/ut-stubs/override_inc) foreach(MODULE ${MODULE_LIST}) if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/src/ut-osapi-${MODULE}.c) - add_library(ut_${SETNAME}_${MODULE} OBJECT src/ut-osapi-${MODULE}.c) - set_target_properties(ut_${SETNAME}_${MODULE} PROPERTIES COMPILE_FLAGS "${UT_C_FLAGS}") + add_library(ut_${SETNAME}_${MODULE} OBJECT + src/ut-osapi-${MODULE}.c + ) + target_compile_options(ut_${SETNAME}_${MODULE} PRIVATE + ${UT_COVERAGE_COMPILE_FLAGS} + ) endif () endforeach() diff --git a/src/unit-test-coverage/ut-stubs/CMakeLists.txt b/src/unit-test-coverage/ut-stubs/CMakeLists.txt index 2a09002f9..aae7fbe9b 100644 --- a/src/unit-test-coverage/ut-stubs/CMakeLists.txt +++ b/src/unit-test-coverage/ut-stubs/CMakeLists.txt @@ -17,8 +17,6 @@ # any coverage test referencing on the shared/ng OSAL layer. # -include_directories(${OSAL_SOURCE_DIR}/src/ut-stubs) - # The "ut_libc_stubs" target provides stub versions of C library calls. # They are prefixed with "OCS_" and target code must be recompiled to # call the OCS_ version of the syscall instead of the regular syscall. @@ -72,6 +70,11 @@ add_library(ut_libc_stubs STATIC EXCLUDE_FROM_ALL src/vxworks-taskVarLib-stubs.c src/vxworks-xbdBlkDev-stubs.c) +target_include_directories(ut_libc_stubs PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/inc +) + + # The "ut_osapi_impl_stubs" provides stub functions for internal # OSAL calls used by or implemented by the shared layer. These # are not public API calls. This is only compiled if used. @@ -84,5 +87,9 @@ add_library(ut_osapi_impl_stubs STATIC EXCLUDE_FROM_ALL src/osapi-select-impl-stubs.c src/osapi-timer-impl-stubs.c) -list(APPEND OSALCOVERAGE_STUB_LIB_LIST ut_osapi_impl_stubs ut_libc_stubs) +list(APPEND OSALCOVERAGE_STUB_LIB_LIST + ut_osapi_impl_stubs + ut_libc_stubs +) set(OSALCOVERAGE_STUB_LIB_LIST ${OSALCOVERAGE_STUB_LIB_LIST} PARENT_SCOPE) + diff --git a/src/unit-test-coverage/vxworks/CMakeLists.txt b/src/unit-test-coverage/vxworks/CMakeLists.txt index 52b75996b..7a1041de5 100644 --- a/src/unit-test-coverage/vxworks/CMakeLists.txt +++ b/src/unit-test-coverage/vxworks/CMakeLists.txt @@ -2,9 +2,6 @@ set(MODULE_LIST osapi osfileapi osfilesys osloader osnetwork osselect ostimer posixio posixfile posixgettime printf) -# This always uses OSAL_OPAQUE_OBJECT_IDS and OSAL_ABSTRACT_FILESYS_TYPES -add_definitions(-DOSAL_OPAQUE_OBJECT_IDS -DOSAL_ABSTRACT_FILESYS_TYPES) - # This unit test is allowed to directly include any internal file in # the respective set under test. include_directories(${OSAL_SOURCE_DIR}/src/os/${SETNAME}) diff --git a/src/unit-test-coverage/vxworks/modules/CMakeLists.txt b/src/unit-test-coverage/vxworks/modules/CMakeLists.txt index 4ef5b121c..8611d9800 100644 --- a/src/unit-test-coverage/vxworks/modules/CMakeLists.txt +++ b/src/unit-test-coverage/vxworks/modules/CMakeLists.txt @@ -19,7 +19,7 @@ # object file to be linked into each test # - Each sub-module is wrapped to divert C library calls to the # stub (OCS) counterpart -# - UT_C_FLAGS are enabled to include any code coverage instrumentation +# - UT_COVERAGE_COMPILE_FLAGS are enabled to include any code coverage instrumentation # the "override_inc" dir contains replacement versions of the C-library include files. include_directories(${OSALCOVERAGE_SOURCE_DIR}/ut-stubs/inc) @@ -27,8 +27,12 @@ include_directories(${OSALCOVERAGE_SOURCE_DIR}/ut-stubs/override_inc) foreach(MODULE ${MODULE_LIST}) if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/src/ut-${MODULE}.c) - add_library(ut_${SETNAME}_${MODULE} OBJECT src/ut-${MODULE}.c) - set_target_properties(ut_${SETNAME}_${MODULE} PROPERTIES COMPILE_FLAGS "${UT_C_FLAGS}") + add_library(ut_${SETNAME}_${MODULE} OBJECT + src/ut-${MODULE}.c + ) + target_compile_options(ut_${SETNAME}_${MODULE} PRIVATE + ${UT_COVERAGE_COMPILE_FLAGS} + ) endif () endforeach() diff --git a/src/unit-tests/CMakeLists.txt b/src/unit-tests/CMakeLists.txt index 26770023a..6cd0f641d 100644 --- a/src/unit-tests/CMakeLists.txt +++ b/src/unit-tests/CMakeLists.txt @@ -1,5 +1,5 @@ # CMake snippet for OSAL unit tests -# +# # NOTE: # This set of unit tests only includes platform support where # one of OSP_ARINC653, _LINUX_OS_ or _VXWORKS_OS_ are defined. @@ -11,19 +11,9 @@ # not necessary and other OS's like RTEMS should work. # -# For VxWorks and RTEMS targets there are still a few slight -# variances that need to be accounted for, mainly in the file -# names and/or directory structures that the test case uses. -set(UT_COMPILEDEFS_vxworks "_VXWORKS_OS_") -set(UT_COMPILEDEFS_rtems "_RTEMS_OS_") -set(UT_COMPILEDEFS_posix "_POSIX_OS_") -if (DEFINED UT_COMPILEDEFS_${OSAL_SYSTEM_OSTYPE}) - add_definitions(-D${UT_COMPILEDEFS_${OSAL_SYSTEM_OSTYPE}}) -endif() - enable_testing() -include_directories(${OSAL_SOURCE_DIR}/ut_assert/inc) +include_directories(${UTASSERT_SOURCE_DIR}/inc) include_directories(inc) add_subdirectory(oscore-test) @@ -32,4 +22,4 @@ add_subdirectory(osfilesys-test) add_subdirectory(osfile-test) add_subdirectory(osnetwork-test) add_subdirectory(ostimer-test) - +