From 8a6a4275972f184d97580af875e78a88c61915e2 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 14 Mar 2023 13:02:06 +0100 Subject: [PATCH 1/2] common: move ASAN and UBSAN build after code coverage check --- utils/docker/run-build.sh | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/utils/docker/run-build.sh b/utils/docker/run-build.sh index 10847f100..c86bd6e1a 100755 --- a/utils/docker/run-build.sh +++ b/utils/docker/run-build.sh @@ -143,7 +143,7 @@ LIB=$(find /lib* -name "*protobuf-c.so*" || true) echo echo "##################################################################" -echo "### Verify build with ASAN and UBSAN ($CC, DEBUG)" +echo "### Verify build and install (in dir: ${PREFIX}) ($CC, DEBUG)" echo "##################################################################" mkdir -p $WORKDIR/build @@ -152,19 +152,30 @@ cd $WORKDIR/build CC=$CC \ $CMAKE .. -DCMAKE_BUILD_TYPE=Debug \ -DTEST_DIR=$TEST_DIR \ - -DBUILD_DEVELOPER_MODE=1 \ - -DDEBUG_USE_ASAN=${CI_SANITS} \ - -DDEBUG_USE_UBSAN=${CI_SANITS} + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DTESTS_COVERAGE=$TESTS_COVERAGE \ + -DBUILD_DEVELOPER_MODE=1 make -j$(nproc) ctest --output-on-failure +sudo_password make -j$(nproc) install + +if [ "$TESTS_COVERAGE" == "1" ]; then + upload_codecov tests +fi + +test_compile_all_examples_standalone + +# Uninstall libraries +cd $WORKDIR/build +sudo_password make uninstall cd $WORKDIR rm -rf $WORKDIR/build echo echo "##################################################################" -echo "### Verify build and install (in dir: ${PREFIX}) ($CC, DEBUG)" +echo "### Verify build with ASAN and UBSAN ($CC, DEBUG)" echo "##################################################################" mkdir -p $WORKDIR/build @@ -173,23 +184,12 @@ cd $WORKDIR/build CC=$CC \ $CMAKE .. -DCMAKE_BUILD_TYPE=Debug \ -DTEST_DIR=$TEST_DIR \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DTESTS_COVERAGE=$TESTS_COVERAGE \ - -DBUILD_DEVELOPER_MODE=1 + -DBUILD_DEVELOPER_MODE=1 \ + -DDEBUG_USE_ASAN=${CI_SANITS} \ + -DDEBUG_USE_UBSAN=${CI_SANITS} make -j$(nproc) ctest --output-on-failure -sudo_password make -j$(nproc) install - -if [ "$TESTS_COVERAGE" == "1" ]; then - upload_codecov tests -fi - -test_compile_all_examples_standalone - -# Uninstall libraries -cd $WORKDIR/build -sudo_password make uninstall cd $WORKDIR rm -rf $WORKDIR/build From 2660e6b71d8e65ef407fecd0e76fc18930048ab1 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 14 Apr 2023 15:47:00 +0200 Subject: [PATCH 2/2] common: move on to the new coverage uploader Move on to the new coverage uploader. The old bash uploader is long EOL now. See: https://github.com/pmem/pmdk/pull/5548 --- .github/workflows/coverage.yml | 44 +++++++++++++++++++ .github/workflows/on_pull_request.yml | 4 +- CMakeLists.txt | 2 +- utils/docker/build.sh | 5 --- ...-gcov-files-and-turn-off-verbose-log.patch | 37 ---------------- utils/docker/images/Dockerfile.ubuntu-latest | 5 --- ...Dockerfile.ubuntu-latest-with-rdma-core-45 | 5 --- utils/docker/images/download-scripts.sh | 32 -------------- utils/docker/run-build.sh | 34 +++----------- 9 files changed, 52 insertions(+), 116 deletions(-) create mode 100644 .github/workflows/coverage.yml delete mode 100644 utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch delete mode 100755 utils/docker/images/download-scripts.sh diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 000000000..bd3bf86b4 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,44 @@ +# +# Builds run on every pull request and every push to the repo. +# + +name: CodeCoverage +on: [push, pull_request] + +env: + GITHUB_REPO: pmem/rpma + # use GitHub Container Registry as a repository of docker images + GH_CR_ADDR: ghcr.io + DOCKER_REPO: ghcr.io/pmem/rpma + # use org's Private Access Token to log in to GitHub Container Registry + GH_CR_USER: ${{ secrets.GH_CR_USER }} + GH_CR_PAT: ${{ secrets.GH_CR_PAT }} + HOST_WORKDIR: /home/runner/work/rpma/rpma + WORKDIR: utils/docker + TYPE: normal + OS: ubuntu + OS_VER: latest-with-rdma-core-45 + CC: gcc + TESTS_COVERAGE: 1 + +jobs: + linux: + name: Linux + runs-on: ubuntu-latest + steps: + - name: Clone the git repo + uses: actions/checkout@v3 + + - name: Pull or rebuild the image + run: cd $WORKDIR && ./pull-or-rebuild-image.sh + + - name: Run the build + run: cd $WORKDIR && ./build.sh + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + gcov: true + root_dir: /home/runner/work/rpma/rpma/ + directory: /home/runner/work/rpma/rpma/ + verbose: true diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml index 5c313d887..b5692170f 100644 --- a/.github/workflows/on_pull_request.yml +++ b/.github/workflows/on_pull_request.yml @@ -24,14 +24,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - CONFIG: ["N=1 OS=ubuntu OS_VER=latest CC=gcc", # to be reverted: TESTS_COVERAGE=1", + CONFIG: ["N=1 OS=ubuntu OS_VER=latest CC=gcc", "N=2 OS=ubuntu OS_VER=latest CC=clang PUSH_IMAGE=1", "N=3 OS=fedora OS_VER=latest CC=gcc PUSH_IMAGE=1", "N=4 OS=fedora OS_VER=latest CC=clang AUTO_DOC_UPDATE=1", "N=5 OS=rockylinux OS_VER=9 CC=gcc PUSH_IMAGE=1", "N=6 OS=rockylinux OS_VER=8 CC=gcc PUSH_IMAGE=1", # Ubuntu-latest with rdma-core v45.0 installed from sources - "N=7 OS=ubuntu OS_VER=latest-with-rdma-core-45 CC=gcc TESTS_COVERAGE=1", + "N=7 OS=ubuntu OS_VER=latest-with-rdma-core-45 CC=gcc", "N=8 OS=ubuntu OS_VER=latest-with-rdma-core-45 CC=clang PUSH_IMAGE=1", # Fedora-latest with rdma-core v45.0 installed from sources "N=9 OS=fedora OS_VER=latest-with-rdma-core-45 CC=gcc PUSH_IMAGE=1", diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ee0dd1e7..7f205d1ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -252,7 +252,7 @@ if(DEBUG_USE_UBSAN) endif() if(TESTS_COVERAGE) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -coverage") + set(CMAKE_C_FLAGS "-O0 --coverage ${CMAKE_C_FLAGS}") endif() if(BUILD_DEVELOPER_MODE) diff --git a/utils/docker/build.sh b/utils/docker/build.sh index b715186c1..b75378565 100755 --- a/utils/docker/build.sh +++ b/utils/docker/build.sh @@ -44,10 +44,6 @@ if [[ "$command" == "" ]]; then esac fi -if [ "$TESTS_COVERAGE" == "1" ]; then - docker_opts="${docker_opts} `bash <(curl -s https://codecov.io/env)`"; -fi - if [ -n "$DNS_SERVER" ]; then DNS_SETTING=" --dns=$DNS_SERVER "; fi if [ "$AUTO_DOC_UPDATE" == "1" ]; then @@ -75,7 +71,6 @@ echo Building ${IMG_VER}-${OS}-${OS_VER} # - working directory set (-w) docker run --privileged=true --name=$containerName -i $TTY \ $DNS_SETTING \ - ${docker_opts} \ --env http_proxy=$http_proxy \ --env https_proxy=$https_proxy \ --env AUTO_DOC_UPDATE=$AUTO_DOC_UPDATE \ diff --git a/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch b/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch deleted file mode 100644 index 7377d07f6..000000000 --- a/utils/docker/images/0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch +++ /dev/null @@ -1,37 +0,0 @@ -From d633d3b0a5f03be280efb80a69b9d5ed4e9c4d56 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= -Date: Tue, 14 Jul 2020 13:58:34 +0200 -Subject: [PATCH] fix generating gcov files and turn-off verbose log - ---- - codecov | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/codecov b/codecov -index e702ecd..0a2f4d8 100755 ---- a/codecov -+++ b/codecov -@@ -1108,9 +1108,9 @@ then - if [ "$ft_gcovout" = "0" ]; - then - # suppress gcov output -- bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -exec $gcov_exe -pb $gcov_arg {} +" >/dev/null 2>&1 || true -+ bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -execdir $gcov_exe -pb $gcov_arg {} \;" >/dev/null 2>&1 || true - else -- bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -exec $gcov_exe -pb $gcov_arg {} +" || true -+ bash -c "find $proj_root -type f -name '*.gcno' $gcov_include $gcov_ignore -execdir $gcov_exe -pb $gcov_arg {} \;" || true - fi - else - say "${e}==>${x} gcov disabled" -@@ -1425,7 +1425,7 @@ do - report_len=$(wc -c < "$file") - if [ "$report_len" -ne 0 ]; - then -- say " ${g}+${x} $file ${e}bytes=$(echo "$report_len" | tr -d ' ')${x}" -+ #say " ${g}+${x} $file ${e}bytes=$(echo "$report_len" | tr -d ' ')${x}" - # append to to upload - _filename=$(basename "$file") - if [ "${_filename##*.}" = 'gcov' ]; --- -2.25.1 - diff --git a/utils/docker/images/Dockerfile.ubuntu-latest b/utils/docker/images/Dockerfile.ubuntu-latest index b7f539848..0189cf5f6 100644 --- a/utils/docker/images/Dockerfile.ubuntu-latest +++ b/utils/docker/images/Dockerfile.ubuntu-latest @@ -69,11 +69,6 @@ RUN ./install-cmocka.sh COPY install-txt2man.sh install-txt2man.sh RUN ./install-txt2man.sh -# Download scripts required in run-*.sh -COPY download-scripts.sh download-scripts.sh -COPY 0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch 0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch -RUN ./download-scripts.sh - # Add user ENV USER user ENV USERPASS p1a2s3s4 diff --git a/utils/docker/images/Dockerfile.ubuntu-latest-with-rdma-core-45 b/utils/docker/images/Dockerfile.ubuntu-latest-with-rdma-core-45 index c0a0942f9..000b0e5fc 100644 --- a/utils/docker/images/Dockerfile.ubuntu-latest-with-rdma-core-45 +++ b/utils/docker/images/Dockerfile.ubuntu-latest-with-rdma-core-45 @@ -80,11 +80,6 @@ RUN ./install-cmocka.sh COPY install-txt2man.sh install-txt2man.sh RUN ./install-txt2man.sh -# Download scripts required in run-*.sh -COPY download-scripts.sh download-scripts.sh -COPY 0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch 0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch -RUN ./download-scripts.sh - # Add user ENV USER user ENV USERPASS p1a2s3s4 diff --git a/utils/docker/images/download-scripts.sh b/utils/docker/images/download-scripts.sh deleted file mode 100755 index 943fbd42b..000000000 --- a/utils/docker/images/download-scripts.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2022, Intel Corporation - -# -# download-scripts.sh - downloads specific version of codecov's bash -# script to generate and upload reports. It's useful, -# since unverified version may break coverage results. -# - -set -e - -# main: Merge pull request #342 from codecov/revert-proj-name-..., 18.08.2020 -CODECOV_VERSION="e877c1280cc6e902101fb5df2981ed1c962da7f0" - -if [ "${SKIP_SCRIPTS_DOWNLOAD}" ]; then - echo "Variable 'SKIP_SCRIPTS_DOWNLOAD' is set; skipping scripts' download" - exit -fi - -mkdir -p /opt/scripts - -# Download codecov's bash script -git clone https://github.com/codecov/codecov-bash -cd codecov-bash -git checkout $CODECOV_VERSION - -git apply ../0001-fix-generating-gcov-files-and-turn-off-verbose-log.patch -mv -v codecov /opt/scripts/codecov - -cd .. -rm -rf codecov-bash diff --git a/utils/docker/run-build.sh b/utils/docker/run-build.sh index c86bd6e1a..07db2763b 100755 --- a/utils/docker/run-build.sh +++ b/utils/docker/run-build.sh @@ -54,32 +54,6 @@ function sudo_password() { echo $USERPASS | sudo -S $* } -function upload_codecov() { - printf "\n$(tput setaf 1)$(tput setab 7)COVERAGE ${FUNCNAME[0]} START$(tput sgr 0)\n" - - # set proper gcov command - clang_used=$($CMAKE -LA -N . | grep CMAKE_C_COMPILER | grep clang | wc -c) - if [[ $clang_used > 0 ]]; then - gcovexe="llvm-cov gcov" - else - gcovexe="gcov" - fi - - # run gcov exe, using their bash (remove parsed coverage files, set flag and exit 1 if not successful) - # we rely on parsed report on codecov.io; the output is quite long, hence it's disabled using -X flag - /opt/scripts/codecov -c -F $1 -Z -x "$gcovexe" -X "gcovout" - - printf "check for any leftover gcov files\n" - leftover_files=$(find . -name "*.gcov" | wc -l) - if [[ $leftover_files > 0 ]]; then - # display found files and exit with error (they all should be parsed) - find . -name "*.gcov" - return 1 - fi - - printf "$(tput setaf 1)$(tput setab 7)COVERAGE ${FUNCNAME[0]} END$(tput sgr 0)\n\n" -} - function compile_example_standalone() { rm -rf $EXAMPLE_TEST_DIR mkdir $EXAMPLE_TEST_DIR @@ -158,15 +132,17 @@ $CMAKE .. -DCMAKE_BUILD_TYPE=Debug \ make -j$(nproc) ctest --output-on-failure -sudo_password make -j$(nproc) install if [ "$TESTS_COVERAGE" == "1" ]; then - upload_codecov tests + exit 0 fi +# install librpma +sudo_password make -j$(nproc) install + test_compile_all_examples_standalone -# Uninstall libraries +# uninstall librpma cd $WORKDIR/build sudo_password make uninstall