From cd2caa58073da39bb58fa7367286c1fd83bc61f8 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Tue, 17 Dec 2024 21:41:52 +0900 Subject: [PATCH 1/7] use :core-devel Signed-off-by: Yutaka Kondo --- .github/workflows/build-and-test-differential.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml index e77aae169..1b143916f 100644 --- a/.github/workflows/build-and-test-differential.yaml +++ b/.github/workflows/build-and-test-differential.yaml @@ -42,7 +42,7 @@ jobs: - -cuda include: - rosdistro: humble - container: ghcr.io/autowarefoundation/autoware:universe-devel + container: ghcr.io/autowarefoundation/autoware:core-devel build-depends-repos: build_depends.repos - container-suffix: -cuda runner: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large @@ -76,7 +76,7 @@ jobs: clang-tidy-differential: needs: build-and-test-differential runs-on: ubuntu-22.04 - container: ghcr.io/autowarefoundation/autoware:universe-devel-cuda + container: ghcr.io/autowarefoundation/autoware:core-devel steps: - name: Set PR fetch depth run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" From ed565684b7f1e1523aba00dd436f03ddd401bfe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Fatih=20C=C4=B1r=C4=B1t?= Date: Tue, 17 Dec 2024 17:02:35 +0300 Subject: [PATCH 2/7] update build-and-test* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: M. Fatih Cırıt --- .github/workflows/build-and-test-daily.yaml | 2 +- .github/workflows/build-and-test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-daily.yaml b/.github/workflows/build-and-test-daily.yaml index 63822f8b1..2546318b2 100644 --- a/.github/workflows/build-and-test-daily.yaml +++ b/.github/workflows/build-and-test-daily.yaml @@ -19,7 +19,7 @@ jobs: - -cuda include: - rosdistro: humble - container: ghcr.io/autowarefoundation/autoware:universe-devel + container: ghcr.io/autowarefoundation/autoware:core-devel build-depends-repos: build_depends.repos steps: - name: Check out repository diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index a9274804e..64faf1edb 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -27,7 +27,7 @@ jobs: - -cuda include: - rosdistro: humble - container: ghcr.io/autowarefoundation/autoware:universe-devel + container: ghcr.io/autowarefoundation/autoware:core-devel build-depends-repos: build_depends.repos steps: - name: Check out repository From b8b34f93ce688190244a060531e50fe14c9d7334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Fatih=20C=C4=B1r=C4=B1t?= Date: Tue, 17 Dec 2024 17:17:13 +0300 Subject: [PATCH 3/7] update the runners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: M. Fatih Cırıt --- .github/workflows/build-and-test-daily.yaml | 2 +- .github/workflows/build-and-test-differential.yaml | 11 ++--------- .github/workflows/build-and-test.yaml | 2 +- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-test-daily.yaml b/.github/workflows/build-and-test-daily.yaml index 2546318b2..17bc5be71 100644 --- a/.github/workflows/build-and-test-daily.yaml +++ b/.github/workflows/build-and-test-daily.yaml @@ -7,7 +7,7 @@ on: jobs: build-and-test-daily: - runs-on: [self-hosted, linux, X64, gpu] + runs-on: ubuntu-24.04 container: ${{ matrix.container }}${{ matrix.container-suffix }} strategy: fail-fast: false diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml index 1b143916f..a5850fac5 100644 --- a/.github/workflows/build-and-test-differential.yaml +++ b/.github/workflows/build-and-test-differential.yaml @@ -30,7 +30,7 @@ jobs: build-and-test-differential: needs: [make-sure-label-is-present, make-sure-require-cuda-label-is-present] if: ${{ needs.make-sure-label-is-present.outputs.result == 'true' }} - runs-on: ${{ matrix.runner }} + runs-on: ubuntu-24.04 container: ${{ matrix.container }}${{ matrix.container-suffix }} strategy: fail-fast: false @@ -44,12 +44,6 @@ jobs: - rosdistro: humble container: ghcr.io/autowarefoundation/autoware:core-devel build-depends-repos: build_depends.repos - - container-suffix: -cuda - runner: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large - build-pre-command: taskset --cpu-list 0-6 - - container-suffix: "" - runner: ubuntu-latest - build-pre-command: "" steps: - name: Set PR fetch depth run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" @@ -70,12 +64,11 @@ jobs: container-suffix: ${{ matrix.container-suffix }} runner: ${{ matrix.runner }} build-depends-repos: ${{ matrix.build-depends-repos }} - build-pre-command: ${{ matrix.build-pre-command }} codecov-token: ${{ secrets.CODECOV_TOKEN }} clang-tidy-differential: needs: build-and-test-differential - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: ghcr.io/autowarefoundation/autoware:core-devel steps: - name: Set PR fetch depth diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 64faf1edb..acf8ef7a2 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -16,7 +16,7 @@ env: jobs: build-and-test: - runs-on: codebuild-autoware-us-east-1-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-large + runs-on: ubuntu-24.04 container: ${{ matrix.container }}${{ matrix.container-suffix }} strategy: fail-fast: false From 776fc66fef9011b5f3290e3c1927a40b5a9ddc55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Fatih=20C=C4=B1r=C4=B1t?= Date: Tue, 17 Dec 2024 17:17:29 +0300 Subject: [PATCH 4/7] run build-and-test sequential and only on push MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: M. Fatih Cırıt --- .github/workflows/build-and-test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index acf8ef7a2..a72ca586b 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -4,11 +4,11 @@ on: push: branches: - main - workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true + # Ensures sequential execution of this workflow + group: ${{ github.workflow }} + cancel-in-progress: false env: CC: /usr/lib/ccache/gcc From 83e7561c19f1711305d447a767d25732d3375e91 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Wed, 18 Dec 2024 20:00:13 +0900 Subject: [PATCH 5/7] remove cuda jobs Signed-off-by: Yutaka Kondo --- .../build-and-test-differential/action.yaml | 14 +------ .github/workflows/build-and-test-daily.yaml | 34 +++------------ .../build-and-test-differential.yaml | 22 ++-------- .github/workflows/build-and-test.yaml | 41 ++++--------------- 4 files changed, 20 insertions(+), 91 deletions(-) diff --git a/.github/actions/build-and-test-differential/action.yaml b/.github/actions/build-and-test-differential/action.yaml index 89893e9f5..4cd671957 100644 --- a/.github/actions/build-and-test-differential/action.yaml +++ b/.github/actions/build-and-test-differential/action.yaml @@ -5,12 +5,6 @@ inputs: rosdistro: description: "" required: true - container: - description: "" - required: true - container-suffix: - description: "" - required: true runner: description: "" required: true @@ -65,12 +59,8 @@ runs: - name: Export CUDA state as a variable for adding to cache key run: | - build_type_cuda_state=nocuda - if [[ "${{ inputs.container-suffix }}" == "-cuda" ]]; then - build_type_cuda_state=cuda - fi - echo "BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" >> "${GITHUB_ENV}" - echo "::notice::BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" + echo "BUILD_TYPE_CUDA_STATE=nocuda" >> "${GITHUB_ENV}" + echo "::notice::BUILD_TYPE_CUDA_STATE=nocuda" shell: bash - name: Build diff --git a/.github/workflows/build-and-test-daily.yaml b/.github/workflows/build-and-test-daily.yaml index 17bc5be71..55925aed8 100644 --- a/.github/workflows/build-and-test-daily.yaml +++ b/.github/workflows/build-and-test-daily.yaml @@ -8,19 +8,7 @@ on: jobs: build-and-test-daily: runs-on: ubuntu-24.04 - container: ${{ matrix.container }}${{ matrix.container-suffix }} - strategy: - fail-fast: false - matrix: - rosdistro: - - humble - container-suffix: - - "" - - -cuda - include: - - rosdistro: humble - container: ghcr.io/autowarefoundation/autoware:core-devel - build-depends-repos: build_depends.repos + container: ghcr.io/autowarefoundation/autoware:core-devel steps: - name: Check out repository uses: actions/checkout@v4 @@ -37,33 +25,23 @@ jobs: id: get-self-packages uses: autowarefoundation/autoware-github-actions/get-self-packages@v1 - - name: Export CUDA state as a variable for adding to cache key - run: | - build_type_cuda_state=nocuda - if [[ "${{ matrix.container-suffix }}" == "-cuda" ]]; then - build_type_cuda_state=cuda - fi - echo "BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" >> "${GITHUB_ENV}" - echo "::notice::BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" - shell: bash - - name: Build if: ${{ steps.get-self-packages.outputs.self-packages != '' }} uses: autowarefoundation/autoware-github-actions/colcon-build@v1 with: - rosdistro: ${{ matrix.rosdistro }} + rosdistro: humble target-packages: ${{ steps.get-self-packages.outputs.self-packages }} - build-depends-repos: ${{ matrix.build-depends-repos }} - cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }} + build-depends-repos: build_depends.repos + cache-key-element: nocuda - name: Test if: ${{ steps.get-self-packages.outputs.self-packages != '' }} id: test uses: autowarefoundation/autoware-github-actions/colcon-test@v1 with: - rosdistro: ${{ matrix.rosdistro }} + rosdistro: humble target-packages: ${{ steps.get-self-packages.outputs.self-packages }} - build-depends-repos: ${{ matrix.build-depends-repos }} + build-depends-repos: build_depends.repos - name: Upload coverage to CodeCov if: ${{ steps.test.outputs.coverage-report-files != '' }} diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml index a5850fac5..6b433e30b 100644 --- a/.github/workflows/build-and-test-differential.yaml +++ b/.github/workflows/build-and-test-differential.yaml @@ -31,19 +31,7 @@ jobs: needs: [make-sure-label-is-present, make-sure-require-cuda-label-is-present] if: ${{ needs.make-sure-label-is-present.outputs.result == 'true' }} runs-on: ubuntu-24.04 - container: ${{ matrix.container }}${{ matrix.container-suffix }} - strategy: - fail-fast: false - matrix: - rosdistro: - - humble - container-suffix: - - "" - - -cuda - include: - - rosdistro: humble - container: ghcr.io/autowarefoundation/autoware:core-devel - build-depends-repos: build_depends.repos + container: ghcr.io/autowarefoundation/autoware:core-devel steps: - name: Set PR fetch depth run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" @@ -56,14 +44,10 @@ jobs: fetch-depth: ${{ env.PR_FETCH_DEPTH }} - name: Run build-and-test-differential action - if: ${{ !(matrix.container-suffix == '-cuda') || needs.make-sure-require-cuda-label-is-present.outputs.result == 'true' }} uses: ./.github/actions/build-and-test-differential with: - rosdistro: ${{ matrix.rosdistro }} - container: ${{ matrix.container }} - container-suffix: ${{ matrix.container-suffix }} - runner: ${{ matrix.runner }} - build-depends-repos: ${{ matrix.build-depends-repos }} + rosdistro: humble + build-depends-repos: build_depends.repos codecov-token: ${{ secrets.CODECOV_TOKEN }} clang-tidy-differential: diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index a72ca586b..6b6c8bff5 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -17,18 +17,7 @@ env: jobs: build-and-test: runs-on: ubuntu-24.04 - container: ${{ matrix.container }}${{ matrix.container-suffix }} - strategy: - fail-fast: false - matrix: - rosdistro: - - humble - container-suffix: - - -cuda - include: - - rosdistro: humble - container: ghcr.io/autowarefoundation/autoware:core-devel - build-depends-repos: build_depends.repos + container: ghcr.io/autowarefoundation/autoware:core-devel steps: - name: Check out repository uses: actions/checkout@v4 @@ -59,9 +48,9 @@ jobs: with: path: | /root/.ccache - key: ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}-${{ github.sha }} + key: ccache-main-${{ runner.arch }}-humble-${{ github.sha }} restore-keys: | - ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}- + ccache-main-${{ runner.arch }}-humble- - name: Limit ccache size run: | @@ -75,47 +64,35 @@ jobs: ccache --zero-stats shell: bash - - name: Export CUDA state as a variable for adding to cache key - run: | - build_type_cuda_state=nocuda - if [[ "${{ matrix.container-suffix }}" == "-cuda" ]]; then - build_type_cuda_state=cuda - fi - echo "BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" >> "${GITHUB_ENV}" - echo "::notice::BUILD_TYPE_CUDA_STATE=$build_type_cuda_state" - shell: bash - - name: Build if: ${{ steps.get-self-packages.outputs.self-packages != '' }} uses: autowarefoundation/autoware-github-actions/colcon-build@v1 with: - rosdistro: ${{ matrix.rosdistro }} + rosdistro: humble target-packages: ${{ steps.get-self-packages.outputs.self-packages }} - build-depends-repos: ${{ matrix.build-depends-repos }} - cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }} + build-depends-repos: build_depends.repos + cache-key-element: nocuda build-pre-command: taskset --cpu-list 0-6 - name: Show ccache stats after build run: du -sh ${CCACHE_DIR} && ccache -s shell: bash - # Only keep save the -cuda version because cuda packages covers non-cuda packages too - name: Push the ccache cache - if: matrix.container-suffix == '-cuda' uses: actions/cache/save@v4 with: path: | /root/.ccache - key: ccache-main-${{ runner.arch }}-${{ matrix.rosdistro }}-${{ github.sha }} + key: ccache-main-${{ runner.arch }}-humble-${{ github.sha }} - name: Test if: ${{ steps.get-self-packages.outputs.self-packages != '' }} id: test uses: autowarefoundation/autoware-github-actions/colcon-test@v1 with: - rosdistro: ${{ matrix.rosdistro }} + rosdistro: humble target-packages: ${{ steps.get-self-packages.outputs.self-packages }} - build-depends-repos: ${{ matrix.build-depends-repos }} + build-depends-repos: build_depends.repos - name: Upload coverage to CodeCov if: ${{ steps.test.outputs.coverage-report-files != '' }} From 10759dfa858dbdb46cf75d3da8dc2d11c354e45a Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Wed, 18 Dec 2024 20:32:16 +0900 Subject: [PATCH 6/7] remove label check Signed-off-by: Yutaka Kondo --- .github/workflows/build-and-test-differential.yaml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml index 6b433e30b..fd962f946 100644 --- a/.github/workflows/build-and-test-differential.yaml +++ b/.github/workflows/build-and-test-differential.yaml @@ -22,13 +22,8 @@ jobs: with: label: tag:run-build-and-test-differential - make-sure-require-cuda-label-is-present: - uses: autowarefoundation/autoware-github-actions/.github/workflows/make-sure-label-is-present.yaml@v1 - with: - label: tag:require-cuda-build-and-test - build-and-test-differential: - needs: [make-sure-label-is-present, make-sure-require-cuda-label-is-present] + needs: make-sure-label-is-present if: ${{ needs.make-sure-label-is-present.outputs.result == 'true' }} runs-on: ubuntu-24.04 container: ghcr.io/autowarefoundation/autoware:core-devel @@ -89,7 +84,7 @@ jobs: target-files: ${{ steps.get-changed-files.outputs.changed-files }} clang-tidy-config-url: https://raw.githubusercontent.com/autowarefoundation/autoware/main/.clang-tidy-ci build-depends-repos: build_depends.repos - cache-key-element: cuda + cache-key-element: nocuda - name: Show disk space after the tasks run: df -h From d94c144f86612eaab3a2f9cf89ac205ab5a69b63 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Wed, 18 Dec 2024 20:36:51 +0900 Subject: [PATCH 7/7] remove cache-key-element Signed-off-by: Yutaka Kondo --- .github/actions/build-and-test-differential/action.yaml | 7 ------- .github/workflows/build-and-test-daily.yaml | 1 - .github/workflows/build-and-test-differential.yaml | 1 - .github/workflows/build-and-test.yaml | 1 - 4 files changed, 10 deletions(-) diff --git a/.github/actions/build-and-test-differential/action.yaml b/.github/actions/build-and-test-differential/action.yaml index 4cd671957..175e71c48 100644 --- a/.github/actions/build-and-test-differential/action.yaml +++ b/.github/actions/build-and-test-differential/action.yaml @@ -57,12 +57,6 @@ runs: ccache --zero-stats shell: bash - - name: Export CUDA state as a variable for adding to cache key - run: | - echo "BUILD_TYPE_CUDA_STATE=nocuda" >> "${GITHUB_ENV}" - echo "::notice::BUILD_TYPE_CUDA_STATE=nocuda" - shell: bash - - name: Build if: ${{ steps.get-modified-packages.outputs.modified-packages != '' }} uses: autowarefoundation/autoware-github-actions/colcon-build@v1 @@ -70,7 +64,6 @@ runs: rosdistro: ${{ inputs.rosdistro }} target-packages: ${{ steps.get-modified-packages.outputs.modified-packages }} build-depends-repos: ${{ inputs.build-depends-repos }} - cache-key-element: ${{ env.BUILD_TYPE_CUDA_STATE }} build-pre-command: ${{ inputs.build-pre-command }} - name: Show ccache stats after build diff --git a/.github/workflows/build-and-test-daily.yaml b/.github/workflows/build-and-test-daily.yaml index 55925aed8..a79bbd623 100644 --- a/.github/workflows/build-and-test-daily.yaml +++ b/.github/workflows/build-and-test-daily.yaml @@ -32,7 +32,6 @@ jobs: rosdistro: humble target-packages: ${{ steps.get-self-packages.outputs.self-packages }} build-depends-repos: build_depends.repos - cache-key-element: nocuda - name: Test if: ${{ steps.get-self-packages.outputs.self-packages != '' }} diff --git a/.github/workflows/build-and-test-differential.yaml b/.github/workflows/build-and-test-differential.yaml index fd962f946..e93751474 100644 --- a/.github/workflows/build-and-test-differential.yaml +++ b/.github/workflows/build-and-test-differential.yaml @@ -84,7 +84,6 @@ jobs: target-files: ${{ steps.get-changed-files.outputs.changed-files }} clang-tidy-config-url: https://raw.githubusercontent.com/autowarefoundation/autoware/main/.clang-tidy-ci build-depends-repos: build_depends.repos - cache-key-element: nocuda - name: Show disk space after the tasks run: df -h diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 6b6c8bff5..52bdc7830 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -71,7 +71,6 @@ jobs: rosdistro: humble target-packages: ${{ steps.get-self-packages.outputs.self-packages }} build-depends-repos: build_depends.repos - cache-key-element: nocuda build-pre-command: taskset --cpu-list 0-6 - name: Show ccache stats after build