From 5d0fc223535b8cfbb4c083b98491552435baa420 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 24 Jan 2024 18:01:04 +0000 Subject: [PATCH 1/2] Per #2796, update versions of actions to fix the node 16 to 20 warning message. --- .../build_docker_and_trigger_metplus.yml | 4 +- .github/workflows/documentation.yml | 8 +-- .github/workflows/testing.yml | 60 +++++++++---------- .github/workflows/update_truth.yml | 2 +- 4 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build_docker_and_trigger_metplus.yml b/.github/workflows/build_docker_and_trigger_metplus.yml index 89e5e14f67..73d64f1d95 100644 --- a/.github/workflows/build_docker_and_trigger_metplus.yml +++ b/.github/workflows/build_docker_and_trigger_metplus.yml @@ -18,7 +18,7 @@ jobs: name: Handle Docker Image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Get branch name id: get_branch_name @@ -47,7 +47,7 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - - uses: actions/github-script@v6 + - uses: actions/github-script@v7 with: github-token: ${{ secrets.METPLUS_BOT_TOKEN }} script: | diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index b039177ff7..9f061df17b 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -18,8 +18,8 @@ jobs: name: Build Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.8' - name: Install dependencies @@ -29,12 +29,12 @@ jobs: python -m pip install -r docs/requirements.txt - name: Build docs run: ./.github/jobs/build_documentation.sh - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: documentation path: artifact/documentation - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: failure() with: name: documentation_warnings.log diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 40c22d80e9..744859b64c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -50,7 +50,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set job controls id: job_status run: .github/jobs/set_job_controls.sh @@ -76,7 +76,7 @@ jobs: needs: job_control if: ${{ needs.job_control.outputs.run_compile == 'true' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Create directories to store output run: mkdir -p ${RUNNER_WORKSPACE}/logs @@ -102,7 +102,7 @@ jobs: - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs path: ${{ runner.workspace }}/logs @@ -136,7 +136,7 @@ jobs: - 'pb2nc madis2nc pcp_combine' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh @@ -149,14 +149,14 @@ jobs: INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }}-all - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit_1a path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs path: ${{ runner.workspace }}/logs @@ -174,7 +174,7 @@ jobs: - 'met_test_scripts mode_multivar mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh @@ -187,14 +187,14 @@ jobs: INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }}-all - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit_1b path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs path: ${{ runner.workspace }}/logs @@ -213,7 +213,7 @@ jobs: - 'ref_config_lead_36' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh @@ -226,14 +226,14 @@ jobs: INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }}-all - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit_rc_leads path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs path: ${{ runner.workspace }}/logs @@ -250,13 +250,13 @@ jobs: - 'ref_config' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh - name: Download ref_config_leads output from artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: unit_rc_leads path: ${{ runner.workspace }}/output @@ -269,14 +269,14 @@ jobs: INPUT_DATA_VERSION: 'none' - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit_rc path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs path: ${{ runner.workspace }}/logs @@ -296,13 +296,13 @@ jobs: - 'ensemble_stat stat_analysis_es' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh - name: Download 1a output from artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: unit_1a path: ${{ runner.workspace }}/output @@ -315,14 +315,14 @@ jobs: INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }}-all - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit_2a path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs path: ${{ runner.workspace }}/logs @@ -342,13 +342,13 @@ jobs: - 'python point2grid plot_data_plane mode mode_analysis perc_thresh hira plot_point_obs quality_filter obs_summary duplicate_flag' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh - name: Download 1a output from artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: unit_1a path: ${{ runner.workspace }}/output @@ -361,14 +361,14 @@ jobs: INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }}-all - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: unit_2b path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs path: ${{ runner.workspace }}/logs @@ -381,14 +381,14 @@ jobs: if: ${{ needs.job_control.outputs.run_diff == 'true' }} steps: - name: Download data from previous jobs - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Copy test output into single directory run: | mkdir ${RUNNER_WORKSPACE}/output cp -r unit_*/* ${RUNNER_WORKSPACE}/output/ - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run Diff Tests in Docker run: .github/jobs/run_diff_docker.sh @@ -399,7 +399,7 @@ jobs: - name: Upload diff files as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: diff path: ${{ runner.workspace }}/diff @@ -407,7 +407,7 @@ jobs: - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs path: ${{ runner.workspace }}/logs @@ -419,13 +419,13 @@ jobs: needs: [job_control, unit_1b, unit_2a, unit_2b, unit_rc] if: ${{ needs.job_control.outputs.run_update_truth == 'true' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh - name: Download data from previous jobs - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Copy test output into single directory run: | diff --git a/.github/workflows/update_truth.yml b/.github/workflows/update_truth.yml index cf9734e449..4b8c310dd7 100644 --- a/.github/workflows/update_truth.yml +++ b/.github/workflows/update_truth.yml @@ -26,7 +26,7 @@ jobs: fi echo ERROR: Branch is $branch_name - must be develop or match main_vX.Y exit 1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Checkout repository with: fetch-depth: 0 From 3dd643922ca74171d6cf47f619c254a8454e4d76 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 25 Jan 2024 22:24:13 +0000 Subject: [PATCH 2/2] Per #2796, port fixes for artifact name handling over from the develop branch to the main_v11.1 testing workflow. Also add the compilation_options.yml workflow since the workflows are being updated. --- .github/workflows/compilation_options.yml | 90 ++++++++++++++++ .github/workflows/testing.yml | 124 ++++++++++++++-------- 2 files changed, 168 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/compilation_options.yml diff --git a/.github/workflows/compilation_options.yml b/.github/workflows/compilation_options.yml new file mode 100644 index 0000000000..83118dcb5e --- /dev/null +++ b/.github/workflows/compilation_options.yml @@ -0,0 +1,90 @@ +name: Compilation Options + +# Test matrix of MET configuration/compilation options + +on: + + # Note that scheduled cron events are run on the default branch. + # Enable this schedule when main_v12.0 becomes the default branch. + + # schedule: + # - cron: '0 7 * * 0'' + + push: + + tags: + - '**' + + workflow_dispatch: + +env: + DOCKERHUB_REPO: dtcenter/met-dev + +jobs: + + job_control: + name: Determine which jobs to run + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set job controls + id: job_status + run: .github/jobs/set_job_controls.sh + env: + commit_msg: ${{ github.event.head_commit.message }} + force_tests: ${{ github.event.inputs.force_tests }} + + outputs: + met_base_repo: ${{ steps.job_status.outputs.met_base_repo }} + met_base_tag: ${{ steps.job_status.outputs.met_base_tag }} + branch_name: ${{ steps.job_status.outputs.branch_name }} + + compile: + name: Config Opts + runs-on: ubuntu-latest + needs: job_control + strategy: + matrix: + include: + - jobid: 'job1' + config: '' + - jobid: 'job2' + config: '--enable-grib2 --enable-python --enable-lidar2nc --enable-mode_graphics --enable-modis' + - jobid: 'job3' + config: '--enable-grib2' + - jobid: 'job4' + config: '--enable-python' + - jobid: 'job5' + config: '--enable-lidar2nc' + - jobid: 'job6' + config: '--enable-mode_graphics' + - jobid: 'job7' + config: '--enable-modis' + fail-fast: false + steps: + - uses: actions/checkout@v4 + + - name: Create directories to store output + run: mkdir -p ${RUNNER_WORKSPACE}/logs + + - name: Compile MET in Docker + run: .github/jobs/build_docker_image.sh + env: + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + MET_BASE_REPO: ${{ needs.job_control.outputs.met_base_repo }} + MET_BASE_TAG: ${{ needs.job_control.outputs.met_base_tag }} + MET_CONFIG_OPTS: ${{ matrix.config }} + + - name: Copy all build log files into logs directory + if: always() + run: cp ${GITHUB_WORKSPACE}/*.log ${RUNNER_WORKSPACE}/logs/ + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: logs_${{ matrix.jobid }} + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 744859b64c..cc8c414fe7 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -104,7 +104,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: logs + name: logs_compile path: ${{ runner.workspace }}/logs if-no-files-found: ignore @@ -131,9 +131,11 @@ jobs: if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'ascii2nc' - - 'pb2nc madis2nc pcp_combine' + include: + - jobid: 'job1' + tests: 'ascii2nc' + - jobid: 'job2' + tests: 'pb2nc madis2nc pcp_combine' fail-fast: false steps: - uses: actions/checkout@v4 @@ -151,14 +153,14 @@ jobs: - name: Upload output as artifact uses: actions/upload-artifact@v4 with: - name: unit_1a + name: unit_1a_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() uses: actions/upload-artifact@v4 with: - name: logs + name: logs_unit_1a_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore @@ -169,9 +171,11 @@ jobs: if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'ascii2nc_indy pb2nc_indy tc_dland tc_pairs tc_stat plot_tc tc_rmw rmw_analysis tc_diag tc_gen' - - 'met_test_scripts mode_multivar mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian' + include: + - jobid: 'job1' + tests: 'ascii2nc_indy pb2nc_indy tc_dland tc_pairs tc_stat plot_tc tc_rmw rmw_analysis tc_diag tc_gen' + - jobid: 'job2' + tests: 'met_test_scripts mode_multivar mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian' fail-fast: false steps: - uses: actions/checkout@v4 @@ -189,28 +193,31 @@ jobs: - name: Upload output as artifact uses: actions/upload-artifact@v4 with: - name: unit_1b + name: unit_1b_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() uses: actions/upload-artifact@v4 with: - name: logs + name: logs_unit_1b_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore - unit_rc_leads: - name: Unit RC leads + unit_1c: + name: Unit 1c runs-on: ubuntu-latest needs: [job_control, update_input_data, compile] if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'ref_config_lead_00 ref_config_lead_12' - - 'ref_config_lead_24 ref_config_lead_48' - - 'ref_config_lead_36' + include: + - jobid: 'job1' + tests: 'ref_config_lead_00 ref_config_lead_12' + - jobid: 'job2' + tests: 'ref_config_lead_24 ref_config_lead_48' + - jobid: 'job3' + tests: 'ref_config_lead_36' fail-fast: false steps: - uses: actions/checkout@v4 @@ -228,26 +235,27 @@ jobs: - name: Upload output as artifact uses: actions/upload-artifact@v4 with: - name: unit_rc_leads + name: unit_1c_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() uses: actions/upload-artifact@v4 with: - name: logs + name: logs_rc_leads_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore - unit_rc: - name: Unit RC + unit_2c: + name: Unit 2c runs-on: ubuntu-latest - needs: [job_control, unit_rc_leads] + needs: [job_control, unit_1c] if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'ref_config' + include: + - jobid: 'job1' + tests: 'ref_config' fail-fast: false steps: - uses: actions/checkout@v4 @@ -258,8 +266,9 @@ jobs: - name: Download ref_config_leads output from artifact uses: actions/download-artifact@v4 with: - name: unit_rc_leads path: ${{ runner.workspace }}/output + pattern: unit_1c_job* + merge-multiple: true - name: Run Unit Tests in Docker run: .github/jobs/run_unit_docker.sh @@ -271,14 +280,14 @@ jobs: - name: Upload output as artifact uses: actions/upload-artifact@v4 with: - name: unit_rc + name: unit_2c_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() uses: actions/upload-artifact@v4 with: - name: logs + name: logs_unit_2c_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore @@ -289,11 +298,15 @@ jobs: if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'point_stat stat_analysis_ps' - - 'grid_stat stat_analysis_gs' - - 'wavelet_stat stat_analysis_ws' - - 'ensemble_stat stat_analysis_es' + include: + - jobid: 'job1' + tests: 'point_stat stat_analysis_ps' + - jobid: 'job2' + tests: 'grid_stat stat_analysis_gs' + - jobid: 'job3' + tests: 'wavelet_stat stat_analysis_ws' + - jobid: 'job4' + tests: 'ensemble_stat stat_analysis_es' fail-fast: false steps: - uses: actions/checkout@v4 @@ -304,8 +317,9 @@ jobs: - name: Download 1a output from artifact uses: actions/download-artifact@v4 with: - name: unit_1a path: ${{ runner.workspace }}/output + pattern: unit_1a_job* + merge-multiple: true - name: Run Unit Tests in Docker run: .github/jobs/run_unit_docker.sh @@ -317,14 +331,14 @@ jobs: - name: Upload output as artifact uses: actions/upload-artifact@v4 with: - name: unit_2a + name: unit_2a_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() uses: actions/upload-artifact@v4 with: - name: logs + name: logs_unit_2a_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore @@ -335,11 +349,15 @@ jobs: if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'climatology_1.0deg' - - 'climatology_1.5deg' - - 'climatology_2.5deg' - - 'python point2grid plot_data_plane mode mode_analysis perc_thresh hira plot_point_obs quality_filter obs_summary duplicate_flag' + include: + - jobid: 'job1' + tests: 'climatology_1.0deg' + - jobid: 'job2' + tests: 'climatology_1.5deg' + - jobid: 'job3' + tests: 'climatology_2.5deg' + - jobid: 'job4' + tests: 'python point2grid plot_data_plane mode mode_analysis perc_thresh hira plot_point_obs quality_filter obs_summary duplicate_flag' fail-fast: false steps: - uses: actions/checkout@v4 @@ -350,8 +368,9 @@ jobs: - name: Download 1a output from artifact uses: actions/download-artifact@v4 with: - name: unit_1a path: ${{ runner.workspace }}/output + pattern: unit_1a_job* + merge-multiple: true - name: Run Unit Tests in Docker run: .github/jobs/run_unit_docker.sh @@ -363,21 +382,21 @@ jobs: - name: Upload output as artifact uses: actions/upload-artifact@v4 with: - name: unit_2b + name: unit_2b_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() uses: actions/upload-artifact@v4 with: - name: logs + name: logs_unit_2b_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore run_diffs: name: Check for Differences runs-on: ubuntu-latest - needs: [job_control, unit_1b, unit_2a, unit_2b, unit_rc] + needs: [job_control, unit_1b, unit_2a, unit_2b, unit_2c] if: ${{ needs.job_control.outputs.run_diff == 'true' }} steps: - name: Download data from previous jobs @@ -409,14 +428,27 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: logs + name: logs_diff path: ${{ runner.workspace }}/logs if-no-files-found: ignore + merge_logs: + name: Merge Log Artifacts + runs-on: ubuntu-latest + needs: [run_diffs] + if: ${{ always() }} + steps: + - name: Upload merged logs as artifact + uses: actions/upload-artifact/merge@v4 + with: + name: logs + pattern: logs_* + delete-merged: true + update_truth: name: Update Truth Data runs-on: ubuntu-latest - needs: [job_control, unit_1b, unit_2a, unit_2b, unit_rc] + needs: [job_control, unit_1b, unit_2a, unit_2b, unit_2c] if: ${{ needs.job_control.outputs.run_update_truth == 'true' }} steps: - uses: actions/checkout@v4