Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature #2796 develop gha_node20, fix artifact names #2799

Merged
merged 8 commits into from
Jan 25, 2024
Merged

Feature #2796 develop gha_node20, fix artifact names #2799

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
97c5982
Per #2796, update actions/checkout@v3 to actions/checkout@v4.
JohnHalleyGotway Jan 24, 2024
e8d4a03
Per #2796, update to actions/upload-artifact@v4 and actions/setup-pyt…
JohnHalleyGotway Jan 24, 2024
34f9a2a
Per #2796, update to download-artifact@v4 and actions/github-script@v7
JohnHalleyGotway Jan 24, 2024
90e0175
Per #2796, update testing workflow for update-artifact v4 version whi…
JohnHalleyGotway Jan 25, 2024
e382140
Per #2796, merge the logs into a single artifact. ci-run-unit
JohnHalleyGotway Jan 25, 2024
ebc3754
Per #2796, renaming rc_leads and rc as 1c and 2c, respectively for co…
JohnHalleyGotway Jan 25, 2024
552fbeb
Per #2796, update the compilation workflow to make the log artifact n…
JohnHalleyGotway Jan 25, 2024
645d2fb
Merge remote-tracking branch 'origin/develop' into feature_2796_devel…
JohnHalleyGotway Jan 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 19 additions & 11 deletions .github/workflows/compilation_options.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,23 @@ jobs:
needs: job_control
strategy:
matrix:
config_opts:
- ''
- '--enable-all'
- '--enable-grib2'
- '--enable-python'
- '--enable-ugrid'
- '--enable-lidar2nc'
- '--enable-mode_graphics'
- '--enable-modis'
include:
- jobid: 'job1'
config: ''
- jobid: 'job2'
config: '--enable-all'
- jobid: 'job3'
config: '--enable-grib2'
- jobid: 'job4'
config: '--enable-python'
- jobid: 'job5'
config: '--enable-ugrid'
- jobid: 'job6'
config: '--enable-lidar2nc'
- jobid: 'job7'
config: '--enable-mode_graphics'
- jobid: 'job8'
config: '--enable-modis'
fail-fast: false
steps:
- uses: actions/checkout@v4
Expand All @@ -68,7 +76,7 @@ jobs:
SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }}
MET_BASE_REPO: ${{ needs.job_control.outputs.met_base_repo }}
MET_BASE_TAG: ${{ needs.job_control.outputs.met_base_tag }}
MET_CONFIG_OPTS: ${{ matrix.config_opts }}
MET_CONFIG_OPTS: ${{ matrix.config }}

- name: Copy all build log files into logs directory
if: always()
Expand All @@ -78,7 +86,7 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: logs
name: logs_${{ matrix.jobid }}
path: ${{ runner.workspace }}/logs
if-no-files-found: ignore

127 changes: 80 additions & 47 deletions .github/workflows/testing.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: logs
name: logs_compile
path: ${{ runner.workspace }}/logs
if-no-files-found: ignore

Expand All @@ -132,9 +132,11 @@ jobs:
if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
strategy:
matrix:
tests:
- 'ascii2nc'
- 'pb2nc madis2nc pcp_combine'
include:
- jobid: 'job1'
tests: 'ascii2nc'
- jobid: 'job2'
tests: 'pb2nc madis2nc pcp_combine'
fail-fast: false
steps:
- uses: actions/checkout@v4
Expand All @@ -152,14 +154,14 @@ jobs:
- name: Upload output as artifact
uses: actions/upload-artifact@v4
with:
name: unit_1a
name: unit_1a_${{ matrix.jobid }}
path: ${{ runner.workspace }}/output

- name: Upload logs as artifact
if: always()
uses: actions/upload-artifact@v4
with:
name: logs
name: logs_unit_1a_${{ matrix.jobid }}
path: ${{ runner.workspace }}/logs
if-no-files-found: ignore

Expand All @@ -170,9 +172,11 @@ jobs:
if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
strategy:
matrix:
tests:
- 'ascii2nc_indy pb2nc_indy tc_dland tc_pairs tc_stat plot_tc tc_rmw rmw_analysis tc_diag tc_gen'
- 'met_test_scripts mode_multivar mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian'
include:
- jobid: 'job1'
tests: 'ascii2nc_indy pb2nc_indy tc_dland tc_pairs tc_stat plot_tc tc_rmw rmw_analysis tc_diag tc_gen'
- jobid: 'job2'
tests: 'met_test_scripts mode_multivar mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian'
fail-fast: false
steps:
- uses: actions/checkout@v4
Expand All @@ -190,28 +194,31 @@ jobs:
- name: Upload output as artifact
uses: actions/upload-artifact@v4
with:
name: unit_1b
name: unit_1b_${{ matrix.jobid }}
path: ${{ runner.workspace }}/output

- name: Upload logs as artifact
if: always()
uses: actions/upload-artifact@v4
with:
name: logs
name: logs_unit_1b_${{ matrix.jobid }}
path: ${{ runner.workspace }}/logs
if-no-files-found: ignore

unit_rc_leads:
name: Unit RC leads
unit_1c:
name: Unit 1c
runs-on: ubuntu-latest
needs: [job_control, update_input_data, compile]
if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
strategy:
matrix:
tests:
- 'ref_config_lead_00 ref_config_lead_12'
- 'ref_config_lead_24 ref_config_lead_48'
- 'ref_config_lead_36'
include:
- jobid: 'job1'
tests: 'ref_config_lead_00 ref_config_lead_12'
- jobid: 'job2'
tests: 'ref_config_lead_24 ref_config_lead_48'
- jobid: 'job3'
tests: 'ref_config_lead_36'
fail-fast: false
steps:
- uses: actions/checkout@v4
Expand All @@ -229,26 +236,27 @@ jobs:
- name: Upload output as artifact
uses: actions/upload-artifact@v4
with:
name: unit_rc_leads
name: unit_1c_${{ matrix.jobid }}
path: ${{ runner.workspace }}/output

- name: Upload logs as artifact
if: always()
uses: actions/upload-artifact@v4
with:
name: logs
name: logs_rc_leads_${{ matrix.jobid }}
path: ${{ runner.workspace }}/logs
if-no-files-found: ignore

unit_rc:
name: Unit RC
unit_2c:
name: Unit 2c
runs-on: ubuntu-latest
needs: [job_control, unit_rc_leads]
needs: [job_control, unit_1c]
if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
strategy:
matrix:
tests:
- 'ref_config'
include:
- jobid: 'job1'
tests: 'ref_config'
fail-fast: false
steps:
- uses: actions/checkout@v4
Expand All @@ -259,8 +267,9 @@ jobs:
- name: Download ref_config_leads output from artifact
uses: actions/download-artifact@v4
with:
name: unit_rc_leads
path: ${{ runner.workspace }}/output
pattern: unit_1c_job*
merge-multiple: true

- name: Run Unit Tests in Docker
run: .github/jobs/run_unit_docker.sh
Expand All @@ -272,14 +281,14 @@ jobs:
- name: Upload output as artifact
uses: actions/upload-artifact@v4
with:
name: unit_rc
name: unit_2c_${{ matrix.jobid }}
path: ${{ runner.workspace }}/output

- name: Upload logs as artifact
if: always()
uses: actions/upload-artifact@v4
with:
name: logs
name: logs_unit_2c_${{ matrix.jobid }}
path: ${{ runner.workspace }}/logs
if-no-files-found: ignore

Expand All @@ -290,12 +299,17 @@ jobs:
if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
strategy:
matrix:
tests:
- 'point_stat stat_analysis_ps'
- 'grid_stat stat_analysis_gs'
- 'wavelet_stat stat_analysis_ws'
- 'ensemble_stat stat_analysis_es'
- 'ugrid'
include:
- jobid: 'job1'
tests: 'point_stat stat_analysis_ps'
- jobid: 'job2'
tests: 'grid_stat stat_analysis_gs'
- jobid: 'job3'
tests: 'wavelet_stat stat_analysis_ws'
- jobid: 'job4'
tests: 'ensemble_stat stat_analysis_es'
- jobid: 'job5'
tests: 'ugrid'
fail-fast: false
steps:
- uses: actions/checkout@v4
Expand All @@ -306,8 +320,9 @@ jobs:
- name: Download 1a output from artifact
uses: actions/download-artifact@v4
with:
name: unit_1a
path: ${{ runner.workspace }}/output
pattern: unit_1a_job*
merge-multiple: true

- name: Run Unit Tests in Docker
run: .github/jobs/run_unit_docker.sh
Expand All @@ -319,14 +334,14 @@ jobs:
- name: Upload output as artifact
uses: actions/upload-artifact@v4
with:
name: unit_2a
name: unit_2a_${{ matrix.jobid }}
path: ${{ runner.workspace }}/output

- name: Upload logs as artifact
if: always()
uses: actions/upload-artifact@v4
with:
name: logs
name: logs_unit_2a_${{ matrix.jobid }}
path: ${{ runner.workspace }}/logs
if-no-files-found: ignore

Expand All @@ -337,11 +352,15 @@ jobs:
if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
strategy:
matrix:
tests:
- 'climatology_1.0deg'
- 'climatology_1.5deg'
- 'climatology_2.5deg'
- 'python point2grid plot_data_plane mode mode_analysis perc_thresh hira plot_point_obs quality_filter obs_summary duplicate_flag'
include:
- jobid: 'job1'
tests: 'climatology_1.0deg'
- jobid: 'job2'
tests: 'climatology_1.5deg'
- jobid: 'job3'
tests: 'climatology_2.5deg'
- jobid: 'job4'
tests: 'python point2grid plot_data_plane mode mode_analysis perc_thresh hira plot_point_obs quality_filter obs_summary duplicate_flag'
fail-fast: false
steps:
- uses: actions/checkout@v4
Expand All @@ -352,8 +371,9 @@ jobs:
- name: Download 1a output from artifact
uses: actions/download-artifact@v4
with:
name: unit_1a
path: ${{ runner.workspace }}/output
pattern: unit_1a_job*
merge-multiple: true

- name: Run Unit Tests in Docker
run: .github/jobs/run_unit_docker.sh
Expand All @@ -365,21 +385,21 @@ jobs:
- name: Upload output as artifact
uses: actions/upload-artifact@v4
with:
name: unit_2b
name: unit_2b_${{ matrix.jobid }}
path: ${{ runner.workspace }}/output

- name: Upload logs as artifact
if: always()
uses: actions/upload-artifact@v4
with:
name: logs
name: logs_unit_2b_${{ matrix.jobid }}
path: ${{ runner.workspace }}/logs
if-no-files-found: ignore

run_diffs:
name: Check for Differences
runs-on: ubuntu-latest
needs: [job_control, unit_1b, unit_2a, unit_2b, unit_rc]
needs: [job_control, unit_1b, unit_2a, unit_2b, unit_2c]
if: ${{ needs.job_control.outputs.run_diff == 'true' }}
steps:
- name: Download data from previous jobs
Expand Down Expand Up @@ -411,14 +431,27 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: logs
name: logs_diff
path: ${{ runner.workspace }}/logs
if-no-files-found: ignore

merge_logs:
name: Merge Log Artifacts
runs-on: ubuntu-latest
needs: [run_diffs]
if: ${{ always() }}
steps:
- name: Upload merged logs as artifact
uses: actions/upload-artifact/merge@v4
with:
name: logs
pattern: logs_*
delete-merged: true

update_truth:
name: Update Truth Data
runs-on: ubuntu-latest
needs: [job_control, unit_1b, unit_2a, unit_2b, unit_rc]
needs: [job_control, unit_1b, unit_2a, unit_2b, unit_2c]
if: ${{ needs.job_control.outputs.run_update_truth == 'true' }}
steps:
- uses: actions/checkout@v4
Expand Down