diff --git a/.github/jobs/build_documentation.sh b/.github/jobs/build_documentation.sh index 3306facee0..5bd00ea174 100755 --- a/.github/jobs/build_documentation.sh +++ b/.github/jobs/build_documentation.sh @@ -25,6 +25,7 @@ if [ -s $warning_file ]; then echo Summary: grep WARNING ${DOCS_DIR}/_build/warnings.log grep ERROR ${DOCS_DIR}/_build/warnings.log + grep CRITICAL ${DOCS_DIR}/_build/warnings.log echo Review this log file or download documentation_warnings.log artifact exit 1 fi diff --git a/.github/jobs/copy_output_to_artifact.sh b/.github/jobs/copy_output_to_artifact.sh new file mode 100755 index 0000000000..ca5847bdb1 --- /dev/null +++ b/.github/jobs/copy_output_to_artifact.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +# Called from .github/workflows/testing.yml +# Creates directory for output data artifact and +# copies output data into directory + +artifact_name=$1 +mkdir -p artifact/${artifact_name} +cp -r ${RUNNER_WORKSPACE}/output/* artifact/${artifact_name}/ diff --git a/.github/jobs/create_dirs_for_database.sh b/.github/jobs/create_dirs_for_database.sh new file mode 100755 index 0000000000..49f3e6f37e --- /dev/null +++ b/.github/jobs/create_dirs_for_database.sh @@ -0,0 +1,16 @@ +#! /bin/bash + +# Create directories used for use case that uses METviewer and METdatadb +# Open up write permissions for these directories so that files can +# be written by METviewer Docker container. +# Called by .github/workflows/testing.yml + +if [ -z "${RUNNER_WORKSPACE}" ]; then + echo "ERROR: RUNNER_WORKSPACE env var must be set" + exit 1 +fi + +mkdir -p $RUNNER_WORKSPACE/mysql +mkdir -p $RUNNER_WORKSPACE/output/metviewer +chmod a+w $RUNNER_WORKSPACE/mysql +chmod a+w $RUNNER_WORKSPACE/output/metviewer diff --git a/.github/jobs/create_output_data_volumes.sh b/.github/jobs/create_output_data_volumes.sh index 6cf11f57d2..38f55b7dea 100755 --- a/.github/jobs/create_output_data_volumes.sh +++ b/.github/jobs/create_output_data_volumes.sh @@ -28,7 +28,7 @@ if [ $? != 0 ]; then exit 1 fi -docker_data_output_dir=ci/docker/docker_data_output +docker_data_output_dir=scripts/docker/docker_data_output success=1 for vol_name in use_cases_*; do diff --git a/.github/jobs/docker_setup.sh b/.github/jobs/docker_setup.sh index 79d0514175..0e67e2b324 100755 --- a/.github/jobs/docker_setup.sh +++ b/.github/jobs/docker_setup.sh @@ -29,9 +29,9 @@ duration=$(( SECONDS - start_seconds )) echo "TIMING: docker pull ${DOCKERHUB_TAG} took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS)" # set DOCKERFILE_PATH that is used by docker hook script get_met_version -export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/ci/docker/Dockerfile +export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/scripts/docker/Dockerfile -MET_TAG=`${GITHUB_WORKSPACE}/ci/docker/hooks/get_met_version` +MET_TAG=`${GITHUB_WORKSPACE}/scripts/docker/hooks/get_met_version` # if MET_FORCE_TAG variable is set and not empty, use that version instead if [ ! -z "$MET_FORCE_TAG" ]; then diff --git a/.github/jobs/run_difference_tests.sh b/.github/jobs/run_difference_tests.sh new file mode 100755 index 0000000000..d363b673d5 --- /dev/null +++ b/.github/jobs/run_difference_tests.sh @@ -0,0 +1,23 @@ +#! /bin/bash + +# Called by .github/workflows/testing.yml +# Runs Python script to perform difference testing on use case output +# to compare output to truth data. +# If any differences were reported, set GHA output var upload_diff +# to true, copy difference files to artifact directory, and return +# non-zero status. If no differences were found, set GHA output var +# upload_diff to false + +matrix_categories=$1 +artifact_name=$2 + +.github/jobs/setup_and_run_diff.py ${matrix_categories} $artifact_name + +if [ "$( ls -A ${RUNNER_WORKSPACE}/diff)" ]; then + echo ::set-output name=upload_diff::true + mkdir -p artifact/diff-${artifact_name} + cp -r ${RUNNER_WORKSPACE}/diff/* artifact/diff-${artifact_name} + exit 1 +fi + +echo ::set-output name=upload_diff::false diff --git a/.github/jobs/save_error_logs.sh b/.github/jobs/save_error_logs.sh new file mode 100755 index 0000000000..0662056729 --- /dev/null +++ b/.github/jobs/save_error_logs.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +# Called from .github/workflows/testing.yml +# Call Python script to copy logs from any use case that contains "ERROR:" +# into directory to create GitHub Actions artifact. +# Sets output variable upload_error_logs to 'true' if errors occurred or +# 'false' if no errors occurred + +.github/jobs/copy_error_logs.py ${RUNNER_WORKSPACE}/output artifact/error_logs +if [ -d "artifact/error_logs" ]; then + echo ::set-output name=upload_error_logs::true +else + echo ::set-output name=upload_error_logs::false +fi diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index b8e05e27da..93ed12e970 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -5,7 +5,6 @@ # a push to determine which jobs to run and which to skip. # set default status for jobs -run_docs=true run_get_image=true run_get_input_data=true run_unit_tests=true @@ -49,7 +48,6 @@ else # check commit messages for skip or force keywords if grep -q "ci-skip-all" <<< "$commit_msg"; then - run_docs=false run_get_image=false run_get_input_data=false run_unit_tests=false @@ -66,18 +64,9 @@ else run_unit_tests=false fi - if grep -q "ci-only-docs" <<< "$commit_msg"; then - run_docs=true - run_get_image=false - run_get_input_data=false - run_unit_tests=false - run_use_cases=false - run_save_truth_data=false - run_diff=false - fi - - if grep -q "ci-only-new-cases" <<< "$commit_msg"; then - run_all_use_cases=false + if grep -q "ci-run-all-cases" <<< "$commit_msg"; then + run_use_cases=true + run_all_use_cases=true fi if grep -q "ci-run-all-diff" <<< "$commit_msg"; then @@ -89,26 +78,8 @@ else run_diff=true fi - if grep -q "ci-run-all-cases" <<< "$commit_msg"; then - run_use_cases=true - run_all_use_cases=true - fi - fi -touch job_control_status -echo run_docs=${run_docs} >> job_control_status -echo run_get_image=${run_get_image} >> job_control_status -echo run_get_input_data=${run_get_input_data} >> job_control_status -echo run_unit_tests=${run_unit_tests} >> job_control_status -echo run_use_cases=${run_use_cases} >> job_control_status -echo run_save_truth_data=${run_save_truth_data} >> job_control_status -echo run_all_use_cases=${run_all_use_cases} >> job_control_status -echo run_diff=${run_diff} >> job_control_status -echo external_trigger=${external_trigger} >> job_control_status -echo Job Control Settings: -cat job_control_status - echo ::set-output name=run_get_image::$run_get_image echo ::set-output name=run_get_input_data::$run_get_input_data echo ::set-output name=run_diff::$run_diff diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 79c0b51e32..7b2080704e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,18 +1,22 @@ name: Testing + on: + push: branches: - develop - develop-ref - - feature_* - - main_* - - bugfix_* + - 'feature_*' + - 'main_*' + - 'bugfix_*' paths-ignore: - - docs/** + - 'docs/**' + pull_request: - types: [opened, reopened, synchronize] + types: [opened, synchronize, reopened] paths-ignore: - - docs/** + - 'docs/**' + workflow_dispatch: inputs: repository: @@ -23,14 +27,16 @@ on: required: true ref: description: 'Branch that triggered event' + required: true actor: description: 'User that triggered the event' pusher_email: description: 'Email address of user who triggered push event' jobs: + event_info: - name: "Trigger: ${{ github.event_name != 'workflow_dispatch' && github.event_name || github.event.inputs.repository }} ${{ github.event_name != 'workflow_dispatch' && 'local' || github.event.inputs.pusher_email }} ${{ github.event_name != 'workflow_dispatch' && 'event' || github.event.inputs.sha }}" + name: "Trigger: ${{ github.event_name != 'workflow_dispatch' && github.event_name || github.event.inputs.repository }} ${{ github.event_name != 'workflow_dispatch' && 'local' || github.event.inputs.actor }} ${{ github.event_name != 'workflow_dispatch' && 'event' || github.event.inputs.sha }}" runs-on: ubuntu-latest steps: - name: Print GitHub values for reference @@ -41,6 +47,15 @@ jobs: job_control: name: Determine which jobs to run runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set job controls + id: job_status + run: .github/jobs/set_job_controls.sh + env: + commit_msg: ${{ github.event.head_commit.message }} + outputs: matrix: ${{ steps.job_status.outputs.matrix }} run_some_tests: ${{ steps.job_status.outputs.run_some_tests }} @@ -49,17 +64,7 @@ jobs: run_diff: ${{ steps.job_status.outputs.run_diff }} run_save_truth_data: ${{ steps.job_status.outputs.run_save_truth_data }} external_trigger: ${{ steps.job_status.outputs.external_trigger }} - steps: - - uses: actions/checkout@v2 - - name: Set job controls - id: job_status - run: .github/jobs/set_job_controls.sh - env: - commit_msg: ${{ github.event.head_commit.message }} - - uses: actions/upload-artifact@v2 - with: - name: job_control_status - path: job_control_status + get_image: name: Docker Setup - Get METplus Image runs-on: ubuntu-latest @@ -76,6 +81,7 @@ jobs: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} #MET_FORCE_TAG: 10.0.0 + update_data_volumes: name: Docker Setup - Update Data Volumes runs-on: ubuntu-latest @@ -83,16 +89,20 @@ jobs: if: ${{ needs.job_control.outputs.run_get_input_data == 'true' }} steps: - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 with: python-version: '3.6' + - name: Install dependencies run: python -m pip install --upgrade pip python-dateutil requests bs4 + - name: Update Data Volumes run: .github/jobs/docker_update_data_volumes.py env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + use_case_tests: name: Use Case Tests runs-on: ubuntu-latest @@ -102,21 +112,24 @@ jobs: fail-fast: false matrix: ${{fromJson(needs.job_control.outputs.matrix)}} steps: + + - uses: actions/checkout@v2 + - name: Create directories for database - run: | - mkdir -p $RUNNER_WORKSPACE/mysql - mkdir -p $RUNNER_WORKSPACE/output/metviewer - chmod a+w $RUNNER_WORKSPACE/mysql - chmod a+w $RUNNER_WORKSPACE/output/metviewer + run: .github/jobs/create_dirs_for_database.sh + - name: Create directory for artifacts run: mkdir -p artifact - - uses: actions/checkout@v2 + - name: Get artifact name id: get-artifact-name run: | artifact_name=`.github/jobs/get_artifact_name.sh ${{ matrix.categories }}` echo ::set-output name=artifact_name::${artifact_name} - - uses: ./.github/actions/run_tests + + # run use case tests + - name: Run Use Cases + uses: ./.github/actions/run_tests id: run_tests with: categories: ${{ matrix.categories }} @@ -125,61 +138,43 @@ jobs: - name: Save error logs id: save-errors if: ${{ always() && steps.run_tests.conclusion == 'failure' && matrix.categories != 'pytests' }} - run: | - .github/jobs/copy_error_logs.py \ - ${RUNNER_WORKSPACE}/output \ - artifact/error_logs - if [ -d "artifact/error_logs" ]; then - echo ::set-output name=upload_error_logs::true - else - echo ::set-output name=upload_error_logs::false - fi + run: .github/jobs/save_error_logs.sh # run difference testing - name: Run difference tests id: run-diff if: ${{ needs.job_control.outputs.run_diff == 'true' && steps.run_tests.conclusion == 'success' && matrix.categories != 'pytests' }} - run: | - artifact_name=${{ steps.get-artifact-name.outputs.artifact_name }} - .github/jobs/setup_and_run_diff.py ${{ matrix.categories }} $artifact_name - if [ "$( ls -A ${RUNNER_WORKSPACE}/diff)" ]; then - echo ::set-output name=upload_diff::true - mkdir -p artifact/diff-${artifact_name} - cp -r ${RUNNER_WORKSPACE}/diff/* artifact/diff-${artifact_name} - exit 1 - else - echo ::set-output name=upload_diff::false - fi + run: .github/jobs/run_difference_tests.sh ${{ matrix.categories }} ${{ steps.get-artifact-name.outputs.artifact_name }} # copy output data to save as artifact - name: Save output data id: save-output if: ${{ always() && steps.run_tests.conclusion != 'skipped' && matrix.categories != 'pytests' }} - run: | - artifact_name=${{ steps.get-artifact-name.outputs.artifact_name }} - mkdir -p artifact/${artifact_name} - cp -r ${RUNNER_WORKSPACE}/output/* artifact/${artifact_name}/ + run: .github/jobs/copy_output_to_artifact.sh ${{ steps.get-artifact-name.outputs.artifact_name }} - - uses: actions/upload-artifact@v2 - name: Upload output data artifact + - name: Upload output data artifact + uses: actions/upload-artifact@v2 if: ${{ always() && steps.run_tests.conclusion != 'skipped' && matrix.categories != 'pytests' }} with: name: ${{ steps.get-artifact-name.outputs.artifact_name }} path: artifact/${{ steps.get-artifact-name.outputs.artifact_name }} - - uses: actions/upload-artifact@v2 - name: Upload error logs artifact + + - name: Upload error logs artifact + uses: actions/upload-artifact@v2 if: ${{ always() && steps.save-errors.outputs.upload_error_logs }} with: name: error_logs path: artifact/error_logs if-no-files-found: ignore - - uses: actions/upload-artifact@v2 - name: Upload difference data artifact + + - name: Upload difference data artifact + uses: actions/upload-artifact@v2 if: ${{ always() && steps.run-diff.outputs.upload_diff == 'true' }} with: name: diff-${{ steps.get-artifact-name.outputs.artifact_name }} path: artifact/diff-${{ steps.get-artifact-name.outputs.artifact_name }} if-no-files-found: ignore + create_output_data_volumes: name: Create Output Docker Data Volumes runs-on: ubuntu-latest diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index fe01f46ca0..9b881b61ea 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -665,220 +665,10 @@ will be used in the final pull request. Add use case to the test suite ------------------------------ -In the METplus repository, there is a text file that contains the list of -all use cases:: - - internal_tests/use_cases/all_use_cases.txt - +The **internal_tests/use_cases/all_use_cases.txt** file in the METplus +repository contains the list of all use cases. Add the new use case to this file so it will be available in -the tests. The file is organized by use case category. Each category starts -a line that following the format:: - - Category: - -where is the name of the use case category. If you are adding a -use case that will go into a new category, you will have to add a new category -definition line to this file and add your new use case under it. Each use case -in that category will be found on its own line after this line. -The use cases can be defined using 3 different formats:: - - :: - :::: - :::::: - -**** - -The index is the number associated with the use case so it can be referenced -easily. The first index number in a new category should be 0. -Each use case added should have an index that is one greater than the previous. - -**::** - -This format should only be used if the use case has only 1 configuration file -and no additional Python package dependencies besides the ones that are -required by the METplus wrappers. is the path of the conf file -used for the use case relative to METplus/parm/use_cases. The filename of the -config file without the .conf extension will be used as the name of the use -case. Example:: - - 6::model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf - -The above example will be named -'PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr' and will run using the -configuration file listed. - -**::::** - -This format is required if the use case contains multiple configuration files. -Instead of forcing the script to guess which conf file should be used as the -name of the use case, you must explicitly define it. The name of the use case -must be separated from the with '::' and each conf file path or -conf variable override must be separated by a comma. Example:: - - 44::GridStat_multiple_config:: met_tool_wrapper/GridStat/GridStat.conf,met_tool_wrapper/GridStat/GridStat_forecast.conf,met_tool_wrapper/GridStat/GridStat_observation.conf - -The above example is named 'GridStat_multiple_config' and uses 3 .conf files. -Use cases with only one configuration file can also use this format is desired. - -**::::::** - -This format is used if there are additional dependencies required to run -the use case such as a different Python environment. - is a list of keywords separated by commas. - -Example:: - - 0::CyclonePlotter::met_tool_wrapper/CyclonePlotter/CyclonePlotter.conf,user_env_vars.MET_PYTHON_EXE=python3:: cycloneplotter_env - -See the next section for more information on valid values to supply as -dependencies. - -Dependencies -^^^^^^^^^^^^ - -Conda Environments -"""""""""""""""""" - -The keywords that end with "_env" are Python environments created in Docker -images using Conda that can be used to run use cases. These images are stored -on DockerHub in dtcenter/metplus-envs and are named with a tag that corresponds -to the keyword without the "_env" suffix. -The environments were created using Docker commands via scripts that are found -in ci/docker/docker_env. Existing keywords that set up Conda environments used -for use cases are: - -* metplotpy_env -* spacetime_env -* xesmf_env -* netcdf4_env -* pygrib_env -* metdatadb_env -* h5py_env -* gempak_env - -Example:: - - spacetime_env - -The above example uses the Conda environment -in dtcenter/metplus-envs:**spacetime** to run a user script. -Note that only one dependency that contains the "_env" suffix can be supplied -to a given use case. - -The **gempak_env** is handled a little differently. It is used if -GempakToCF.jar is needed for a use case to convert GEMPAK data to NetCDF -format so it can be read by the MET tools. Instead of creating a Python -environment to use for the use case, this Docker image installs Java and -obtains the GempakToCF.jar file. When creating the Docker container to run -the use cases, the necessary Java files are copied over into the container -that runs the use cases so that the JAR file can be run by METplus wrappers. - -Other Keywords -"""""""""""""" - -Besides specifying Python environments, -there are additional keywords that can be used to set up the environment -to run a use case: - -* **py_embed** - Used if a different Python environment is required to - run a Python Embedding script. If this keyword is included with a Python - environment, then the MET_PYTHON_EXE environment variable will be set to - specify the version of Python3 that is included in that environment - -Example:: - - pygrib_env,py_embed - -In this example, the dtcenter/metplus-envs:**pygrib** environment is used to -run the use case. Since **py_embed** is also included, then the following will -be added to the call to run_metplus.py so that the Python embedding script -will use the **pygrib** environment to run:: - - user_env_vars.MET_PYTHON_EXE=/usr/local/envs/pygrib/bin/python3 - -Please see the MET User's Guide for more information on how to use Python -Embedding. - -* **metviewer** - Used if METviewer should be made available to the use case. - This is typically added for a METdbLoad use case that needs to populate a - database with MET output. - -* **metplus** - Used if a user script needs to call utility functions from the - metplus Python package. This keyword simply adds the METplus source code - directory to the PYTHONPATH so that the metplus.util functions can be - imported. Note that this keyword is not needed unless a different Python - environment is specified with a "_env" keyword. The version of Python that - is used to run typical use cases has already installed the METplus Python - package in its environment, so the package can be imported easily. - - -Creating New Python Environments -"""""""""""""""""""""""""""""""" - -In METplus v4.0.0 and earlier, a list of Python packages were added to use -cases that required additional packages. These packages were either installed -with pip3 or using a script. This approach was very time consuming as some -packages take a very long time to install in Docker. The new approach involves -creating Docker images that use Conda to create a Python environment that can -run the use case. To see what is available in each of the existing Python -environments, refer to the comments in the scripts found in -**ci/docker/docker_env/scripts**. New environments must be added by a METplus -developer, so please contact MET Help if none of these environments contain the -package requirements needed to run a new use case. - -A README file can be found in the ci/docker/docker_env directory that -provides commands that can be run to recreate a Docker image if the -conda environment needs to be updated. Please note that Docker must -be installed on the workstation used to create new Docker images and -a DockerHub account with access to the dtcenter repositories must -be used to push Docker images to DockerHub. - -The README file also contains commands to create a conda environment -that is used for the tests locally. Any base conda environments, -such as metplus_base and py_embed_base, must be created locally first -before creating an environment that builds upon these environments. -Please note that some commands in the scripts are specific to -the Docker environment and may need to be rerun to successfully -build the environment locally. - -**Installing METplus Components** - -These scripts -do not install any METplus components, -such as metplotpy/metcalcpy/metplus, in the Python environment that -may be needed for a use case. This is done because the automated tests -will install and use the latest version (develop) of the packages to -ensure that any changes to those components do not break any existing -use cases. These packages will need to be installed by the user -and need to be updated manually. To install these packages, -activate the Conda environment, obtain the source code from GitHub, -and run "pip3 install ." in the top level directory of the repository. - -Example:: - - conda activate weatherregime - git clone git@github.com:dtcenter/METplotpy - cd METplotpy - git checkout develop - git pull - pip3 install . - -**Cartopy Shapefiles** - -The cartopy python package automatically attempts to download -shapefiles as needed. -The URL that is used in cartopy version 0.18.0 and earlier no longer -exists, so use cases that needs these files will fail if they are -not found locally. If a conda environment uses cartopy, these -shapefiles may need to be downloaded by the user running the use case -even if the conda environment was created by another user. -Cartopy provides a script that can be used to obtain these shapefiles -from the updated URL:: - - wget https://raw.githubusercontent.com/SciTools/cartopy/master/tools/cartopy_feature_download.py - python3 cartopy_feature_download.py cultural physical cultural-extra - - +the tests. See the :ref:`cg-ci-all-use-cases` section for details. .. _add_new_category_to_test_runs: @@ -887,11 +677,12 @@ Add new category to test runs The **.github/parm/use_case_groups.json** file in the METplus repository contains a list of the use case groups to run together. -In METplus version 4.0.0 and earlier, this list was -found in the .github/workflows/testing.yml file. Add a new entry to the list that includes the category of the new use case, the list of indices that correspond to the index number described in the :ref:`add_use_case_to_test_suite` section. + +See the :ref:`cg-ci-use-case-groups` section for details. + Set the "run" variable to true so that the new use case group will run in the automated test suite whenever a new change is pushed to GitHub. This allows users to test that the new use case runs successfully. @@ -910,6 +701,7 @@ Example:: This example adds a new use case group that contains the climate use case with index 2 and is marked to "run" for every push. + New use cases are added as a separate item to make reviewing the test results easier. A new use case will produce new output data that is not found in the "truth" data set which is compared the output of the use case runs to check @@ -919,66 +711,6 @@ It also makes it easier to check the size of the output data and length of time the use case takes to run to determine if it can be added to an existing group or if it should remain in its own group. - -.. _subset_category: - -Subset Category into Multiple Tests -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Use cases can be separated into multiple test jobs. -In the "index_list" value, define the cases to run for the job. -Use cases are numbered starting with 0 and are in order of how they are -found in the all_use_cases.txt file. - -The argument supports a comma-separated list of numbers. Example:: - - { - "category": "data_assimilation", - "index_list": "0,2,4", - "run": false - }, - { - "category": "data_assimilation", - "index_list": "1,3", - "run": false - }, - -The above example will run a job with data_assimilation use cases 0, 2, and -4, then another job with data_assimilation use cases 1 and 3. - -It also supports a range of numbers separated with a dash. Example:: - - { - "category": "data_assimilation", - "index_list": "0-3", - "run": false - }, - { - "category": "data_assimilation", - "index_list": "4-5", - "run": false - }, - -The above example will run a job with data_assimilation 0, 1, 2, and 3, then -another job with data_assimilation 4 and 5. - -You can also use a combination of commas and dashes to define the list of cases -to run. Example:: - - { - "category": "data_assimilation", - "index_list": "0-2,4", - "run": false - }, - { - "category": "data_assimilation", - "index_list": "3", - "run": false - }, - -The above example will run data_assimilation 0, 1, 2, and 4 in one -job, then data_assimilation 3 in another job. - Monitoring Automated Tests -------------------------- @@ -1077,8 +809,8 @@ so that it runs in a reasonable time frame. If the new use case runs in a reasonable amount of time but the total time to run the set of use cases is now above 20 minutes or so, consider creating a -new job for the new use case. See the :ref:`subset_category` section and the -multiple medium_range jobs for an example. +new job for the new use case. See the :ref:`cg-ci-subset_category` section +and the multiple medium_range jobs for an example. .. _exceeded-Github-Actions: diff --git a/docs/Contributors_Guide/continuous_integration.rst b/docs/Contributors_Guide/continuous_integration.rst index f10f62abeb..16a43b1934 100644 --- a/docs/Contributors_Guide/continuous_integration.rst +++ b/docs/Contributors_Guide/continuous_integration.rst @@ -1,12 +1,11 @@ +********************** Continuous Integration -====================== - -More information on Continuous Integration (CI) coming soon! +********************** METplus utilizes GitHub Actions to run processes automatically when changes are pushed to GitHub. These tasks include: -* Building documentation +* Building documentation to catch warnings/errors * Building a Docker image to run tests * Creating/Updating Docker data volumes with new input data used for tests * Running unit tests @@ -14,102 +13,378 @@ are pushed to GitHub. These tasks include: * Comparing use case output to truth data * Creating/Updating Docker data volumes with truth data to use in comparisons -Workflow Control ----------------- - -GitHub Actions is controlled by a file in the .github/workflow directory called -testing.yml. If this file exists and is valid (no errors), GitHub Actions will -read this file and trigger a workflow run if the triggering criteria is met. -It can run multiple jobs in parallel or serially depending on dependency rules -that can be set. Each job can run a series of commands or scripts called steps. -Job steps can include "actions" with can be used to perform tasks. Many useful -actions are provided by GitHub and external collaborators. Developers can also -write their own custom actions to perform complex tasks to simplify a workflow. +GitHub Actions Workflows +======================== + +GitHub Actions runs workflows defined by files in the **.github/workflows** +directory of a GitHub repository. +Files with the .yml suffix are parsed and GitHub Actions will +trigger a workflow run if the triggering criteria is met. +Multiple workflows may be triggered by a single event. +All workflow runs can be seen on the Actions tab of the repository. +Each workflow run is identified by the branch for which it was invoked +as well as the corresponding commit message on that branch. +In general, a green check mark indicates that all checks for +that workflow run passed. +A red X indicates that at least one of the jobs failed. + +Workflows can run multiple jobs in parallel or serially depending on +dependency rules that can be set. +Each job can run a series of commands or scripts called steps. +Steps can include actions which can be used to perform common tasks. +Many useful actions are provided by GitHub and external collaborators. +Developers can also write their own custom actions to perform complex tasks +to simplify a workflow. + +**TODO Add screenshots** + +Testing (testing.yml) +--------------------- + +This workflow performs a variety of tasks to ensure that changes do not break +any existing functionality. +See the :ref:`cg-ci-testing-workflow` for more information. + +Documentation (documentation.yml) +--------------------------------- + +METplus documentation is written using Sphinx. +The METplus components utilize ReadTheDocs to build and display documentation. +However, ReadTheDocs will render the documentation when warnings occur. +This GitHub Actions workflow is run to catch/report warnings and errors. + +This workflow is only triggered when changes are made to files under the +**docs** directory of the METplus repository. +It builds the documentation by running "make clean html" and +makes the files available to download at the end of the workflow +as a GitHub Actions artifact. This step is no longer mandatory because +ReadTheDocs is configured to automatically generate the documentation for each +branch/tag and publish it `online `_. + +The Makefile that runs sphinx-build was modified to write warnings and errors +to a file called warnings.log using the -w argument. This file will be empty +if no errors or warnings have occurred in the building of the documentation. +If it is not empty, the script called by this workflow will exit with a +non-zero value so that the workflow reports a failure. + +.. figure:: figure/ci-doc-error.png + +A summary of the lines that contain WARNING or ERROR are output in the +GitHub Actions log for easy access. +The warnings.log file is also made available as a GitHub Actions +artifact so it can be downloaded and reviewed. Artifacts can be found +at the bottom of the workflow summary page when the workflow has completed. + +.. figure:: figure/ci-doc-artifacts.png + + +Release Published (release_published.yml) - DEPRECATED +------------------------------------------------------ + +**This workflow is no longer be required, as Slack now has GitHub integration +to automatically create posts on certain events.** The workflow YAML file +is still found in the repository for reference, but the workflow has been +disabled via the Actions tab of the METplus GitHub webpage. + +This workflow is triggered when a release is published on GitHub. +It uses cURL to trigger a Slack message on the DTC-METplus announcements +channel that lists information about the release. A Slack bot was created +through the Slack API and the webhook that generated for the Slack channel +was saved as a GitHub Secret. + +.. _cg-ci-testing-workflow: + +Testing Workflow +================ Name -^^^^ +---- The name of a workflow can be specified to describe an overview of what is run. -Currently METplus only has 1 workflow, but others can be added. The following -line in the testing.yml file:: +The following line in the testing.yml file:: - name: METplus CI/CD Workflow + name: Testing -defines the workflow that runs all of the jobs. +defines the workflow identifier that can be seen from the Actions tab on the +METplus GitHub page. .. figure:: figure/gha-workflow-name.png Event Control -^^^^^^^^^^^^^ +------------- + +The **on** keyword defines which events trigger the workflow +to run. There are currently 3 types of events that trigger this workflow: +**push**, **pull_request**, and **workflow_dispatch**. +The jobs that are run in this workflow depend on which event has triggered it. +Many jobs are common to multiple events. +To avoid creating multiple workflow .yml files that contain redundant jobs, +an additional layer of control is added within this workflow. +See :ref:`cg-ci-job-control` for more information. + +Push +^^^^ -The "on" keyword is used to determine which events will trigger the workflow -to run:: +:: on: + push: branches: - develop - develop-ref - - feature_* - - main_* - - bugfix_* + - 'feature_*' + - 'main_*' + - 'bugfix_*' + paths-ignore: + - 'docs/**' + +This configuration tells GitHub Actions to trigger the workflow when changes +are pushed to the repository and the following criteria are met: + +* The branch is named **develop** or **develop-ref** +* The branch starts with **feature\_**, **main\_**, or **bugfix\_** +* Changes were made to at least one file that is not in the **docs** directory. + +Pull Request +^^^^^^^^^^^^ + +:: + pull_request: types: [opened, reopened, synchronize] + paths-ignore: + - 'docs/**' + +This configuration tells GitHub Actions to trigger the workflow for +pull requests in the repository and the following criteria are met: -This configuration tells GitHub Actions to trigger the workflow when: +* The pull request was opened, reopened, or synchronized. +* Changes were made to at least one file that is not in the **docs** directory. -* A push event occurs on the develop or develop-ref branch -* A push event occurs on a branch that starts with - feature\_, main\_, or bugfix\_ -* A pull request is opened, reopened, or synchronized (when new changes are - pushed to the source branch of the pull request. +The **synchronize** type triggers a workflow for every push to a branch +that is included in an open pull request. +If changes were requested in the pull request review, +a new workflow will be triggered for each push. +To prevent many workflows from being triggered, +developers are encouraged to limit the number of pushes for open pull requests. +Note that pull requests can be closed until the necessary changes are +completed, or :ref:`cg-ci-commit-message-keywords` can be used +to suppress the testing workflow. + + +Workflow Dispatch +^^^^^^^^^^^^^^^^^ + +:: + + workflow_dispatch: + inputs: + repository: + description: 'Repository that triggered workflow' + required: true + sha: + description: 'Commit hash that triggered the event' + required: true + ref: + description: 'Branch that triggered event' + required: true + actor: + description: 'User that triggered the event' + + +This configuration enables manual triggering of this workflow. +It allows other GitHub repositories such as MET, METplotpy, and METcalcpy +to trigger this workflow. +It lists the input values that are passed from the external repository. +The inputs include: + +* The repository that triggered the workflow, such as dtcenter/MET +* The commit hash in the external repository that triggered the event +* The reference (or branch) that triggered the event, such as + refs/heads/develop +* The GitHub username that triggered the event in the external repository + (optional) + +The MET, METcalcpy, and METplotpy repositories are configured to +trigger this workflow since they are used in 1 or more METplus use cases. +Currently all 3 repositories only trigger when changes are pushed to their +develop branch. + +Future work is planned to support main_v* branches, which +will involve using the 'ref' input to determine what to obtain in the workflow. +For example, changes pushed to dtcenter/MET main_v10.1 should trigger a +testing workflow that runs on the METplus main_v4.1 branch. Jobs -^^^^ +---- + +The **jobs** keyword is used to define the jobs that are run in the workflow. +Each item under **jobs** is a string that defines the ID of the job. +This value can be referenced within the workflow as needed. +Each job in the testing workflow is described in its own section. + +* :ref:`cg-ci-event-info` +* :ref:`cg-ci-job-control` +* :ref:`cg-ci-get-image` +* :ref:`cg-ci-update-data-volumes` +* :ref:`cg-ci-use-case-tests` +* :ref:`cg-ci-create-output-data-volumes` + +.. _cg-ci-event-info: + +Event Info +---------- + +This job contains information on what triggered the workflow. +The name of the job contains complex logic to cleanly display information +about an event triggered by an external repository when that occurs. +Otherwise, it simply lists the type of local event (push or pull_request) +that triggered the workflow. + +**Insert images of examples of the Trigger job name for local and external** + +It also logs all of the information contained in the 'github' object that +includes all of the available information from the event that triggered +the workflow. This is useful to see what information is available to use +in the workflow based on the event. + +**Insert image of screenshot of the github.event info** -The "jobs" keyword is used to define the jobs that are run in the workflow. -Each item under "jobs" is a string that defines the ID of the job. This value -can be referenced within the workflow as needed. +.. _cg-ci-job-control: Job Control ----------- +:: + + job_control: + name: Determine which jobs to run + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set job controls + id: job_status + run: .github/jobs/set_job_controls.sh + env: + commit_msg: ${{ github.event.head_commit.message }} + + outputs: + matrix: ${{ steps.job_status.outputs.matrix }} + run_some_tests: ${{ steps.job_status.outputs.run_some_tests }} + run_get_image: ${{ steps.job_status.outputs.run_get_image }} + run_get_input_data: ${{ steps.job_status.outputs.run_get_input_data }} + run_diff: ${{ steps.job_status.outputs.run_diff }} + run_save_truth_data: ${{ steps.job_status.outputs.run_save_truth_data }} + external_trigger: ${{ steps.job_status.outputs.external_trigger }} + +This job runs a script called **set_job_controls.sh** +that parses environment variables set by GitHub Actions to determine which +jobs to run. There is :ref:`cg-ci-default-behavior` based on the event that +triggered the workflow and the branch name. +The last commit message before a push event is also parsed to look for +:ref:`cg-ci-commit-message-keywords` that can override the default behavior. + +The script also calls another script called **get_use_cases_to_run.sh** that +reads a JSON file that contains the use case test groups. +The job control settings determine which of the use case groups to run. +See :ref:`cg-ci-use-case-groups` for more information. + +Output Variables +^^^^^^^^^^^^^^^^ + +The step that calls the job control script is given an identifier using the +**id** keyword:: + + id: job_status + run: .github/jobs/set_job_controls.sh + +Values from the script are set as output variables using the following syntax:: + + echo ::set-output name=run_get_image::$run_get_image + +In this example, an output variable named *run_get_image* +(set with **name=run_get_image**) is created with the value of a +variable from the script with the same name (set after the :: characters). +The variable can be referenced elsewhere within the job using the following +syntax:: + + ${{ steps.job_status.outputs.run_get_image }} + +The ID of the step is needed to reference the outputs for that step. +**Note that this notation should be referenced directly in the workflow YAML +file and not inside a script that is called by the workflow.** + +To make the variable available to other jobs in the workflow, it will need +to be set in the **outputs** section of the job:: + + outputs: + run_get_image: ${{ steps.job_status.outputs.run_get_image }} + +The variable **run_get_image** can be referenced by other jobs that include +**job_status** as a job that must complete before starting using the **needs** +keyword:: + + get_image: + name: Docker Setup - Get METplus Image + runs-on: ubuntu-latest + needs: job_control + if: ${{ needs.job_control.outputs.run_get_image == 'true' }} + +Setting **needs: job_control** tells the **get_image** job to wait until the +**job_control** job has completed before running. Since this is the case, this +job can reference output from that job in the **if** value to determine if the +job should be run or not. + +.. _cg-ci-default-behavior: + Default Behavior ^^^^^^^^^^^^^^^^ On Push """"""" -When a push to a feature\_\*, bugfix\_\*, main_v\*, or develop\* branch occurs -the default behavior is to run the following: +When a push event occurs the default behavior is to run the following: -* Build documentation -* Update Docker image +* Create/Update the METplus Docker image * Look for new input data * Run unit tests -* Run any **new** use cases +* Run any use cases marked to run (see :ref:`cg-ci-use-case-tests`) + +If the push is on the **develop** or a **main_vX.Y** branch, then all +of the use cases are run. + +Default behavior for push events can be overridden using +:ref:`cg-ci-commit-message-keywords`. On Pull Request """"""""""""""" -When a pull request is created into the develop branch or a main_v\* branch, -additional jobs are run in automation. In addition to the jobs run for a push, -the scripts will: +When a pull request is created into the **develop** branch or +a **main_vX.Y** branch, additional jobs are run in automation. +In addition to the jobs run for a push, the scripts will: * Run all use cases * Compare use case output to truth data +.. _cg-ci-push-reference-branch: + On Push to Reference Branch """"""""""""""""""""""""""" -Branches with a name that ends with "-ref" contain the state of the repository -that will generate output that is considered "truth" data. -In addition to the jobs run for a normal push, the scripts will: +Branches with a name that ends with **-ref** contain the state of the +repository that will generate output that is considered "truth" data. +In addition to the jobs run for a push, the scripts will: * Run all use cases * Create/Update Docker data volumes that store truth data with the use case output +See :ref:`cg-ci-create-output-data-volumes` for more information. + +.. _cg-ci-commit-message-keywords: + Commit Message Keywords ^^^^^^^^^^^^^^^^^^^^^^^ @@ -119,12 +394,33 @@ Here is a list of the currently supported keywords and what they control: * **ci-skip-all**: Don't run anything - skip all automation jobs * **ci-skip-use-cases**: Don't run any use cases +* **ci-skip-unit-tests**: Don't run the Pytest unit tests * **ci-run-all-cases**: Run all use cases * **ci-run-diff**: Obtain truth data and run diffing logic for use cases that are marked to run * **ci-run-all-diff**: Obtain truth data and run diffing logic for all use cases -* **ci-only-docs**: Only run build documentation job - skip the rest + +.. _cg-ci-get-image: + +Create/Update METplus Docker Image +---------------------------------- + +This job calls the **docker_setup.sh** script. +This script builds a METplus Docker image and pushes it to DockerHub. +The image is pulled instead of built in each test job to save execution time. +The script attempts to pull the appropriate Docker image from DockerHub +(dtcenter/metplus-dev:*BRANCH_NAME*) if it already exists so that unchanged +components of the Docker image do not need to be rebuilt. +This reduces the time it takes to rebuild the image for a given branch on +a subsequent workflow run. + +DockerHub Credentials +^^^^^^^^^^^^^^^^^^^^^ + +The credentials needed to push images to DockerHub are stored in Secret +Environment Variables for the repository. These variables are passed +into the script that needs them using the **env** keyword. Force MET Version Used for Tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -137,10 +433,10 @@ i.e. METplotpy or METviewer, until a corresponding change is made to that component. If this occurs then some of the METplus use cases may break. To allow the tests to run successfully in the meantime, an option was added to force the version of the MET tag that is used to build the METplus Docker image -that is used for testing. In the testing.yml GitHub Actions workflow file -(found in .github/workflows), there is a commented variable called +that is used for testing. In the testing.yml workflow file, +there is a commented variable called MET_FORCE_TAG that can be uncommented and set to force the version of MET to -use. This variable is found in the "get_image" job under the "env" section +use. This variable is found in the **get_image** job under the **env** section for the step named "Get METplus Image." :: @@ -151,3 +447,524 @@ for the step named "Get METplus Image." DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} #MET_FORCE_TAG: 10.0.0 + + +.. _cg-ci-update-data-volumes: + +Create/Update Docker Data Volumes +--------------------------------- + +The METplus use case tests obtain input data from Docker data volumes. +Each use case category that corresponds to a directory in +**parm/use_cases/model_applications** has its own data volume that contains +all of the data needed to run those use cases. The MET Tool Wrapper use cases +found under **parm/use_cases/met_tool_wrapper** also have a data volume. +These data are made available on the DTC web server. + +The logic in this +job checks if the tarfile that contains the data for a use case category has +changed since the corresponding Docker data volume has been last updated. +If it has, then the Docker data volume is regenerated with the new data. + +When new data is needed for a new METplus use case, a directory that is named +after a feature branch is populated with the existing data for the use case +category and the new data is added there. This data is used for testing the +new use case in the automated tests. When the pull request for the new use +case is approved, the new data is moved into the version of the +data that corresponds to the upcoming release (i.e. v4.1) +so that it will be available for future tests. More details on this +process can be found in the :ref:`use_case_input_data` section of the +Add Use Cases chapter of the Contributor's Guide. + + +.. _cg-ci-use-case-tests: + +Use Case Tests +-------------- + +.. _cg-ci-all-use-cases: + +All Use Cases +^^^^^^^^^^^^^ + +All of the existing use cases are listed in **all_use_cases.txt**, +found in internal_tests/use_cases. + +The file is organized by use case category. Each category starts +a line that following the format:: + + Category: + +where ** is the name of the use case category. +See :ref:`use_case_categories` for more information. If you are adding a +use case that will go into a new category, you will have to add a new category +definition line to this file and add your new use case under it. Each use case +in that category will be found on its own line after this line. +The use cases can be defined using the following formats:: + + :::: + :::::: + +index +""""" + +The index is the number associated with the use case so it can be referenced +easily. The first index number in a new category should be 0. +Each use case added should have an index that is one greater than the previous. +If it has been determined that a use case cannot run in the automated tests, +then the index number should be replaced with "#X" so that is it included +in the list for reference but not run by the tests. + +name +"""" + +This is the string identifier of the use case. The name typically matches +the use case configuration filename without the **.conf** extension. + +Example:: + + PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr + + +config_args +""""""""""" + +This is the path of the config file used for the use case relative to +**parm/use_cases**. + +Example:: + + model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf + +If the use case contains multiple configuration files, +they can be listed separated by commas. + +Example:: + + met_tool_wrapper/GridStat/GridStat.conf,met_tool_wrapper/GridStat/GridStat_forecast.conf,met_tool_wrapper/GridStat/GridStat_observation.conf + + +dependencies +"""""""""""" + +If there are additional dependencies required to run the use case, +such as a different Python environment, a list of keywords separated by commas +can be provided. +The :ref:`cg-ci-use-case-dependencies` section contains information +on the keywords that can be used. + +Example:: + + cycloneplotter_env + + +.. _cg-ci-use-case-dependencies: + +Use Case Dependencies +^^^^^^^^^^^^^^^^^^^^^ + +Conda Environments +"""""""""""""""""" + +The keywords that end with **_env** are Python environments created in Docker +images using Conda that can be used to run use cases. These images are stored +on DockerHub in *dtcenter/metplus-envs* and are named with a tag that +corresponds to the keyword without the **_env** suffix. +The environments were created using Docker commands via scripts that are found +in **scripts/docker/docker_env**. +Existing keywords that set up Conda environments used for use cases are: + +* cfgrib_env +* h5py_env +* icecover_env +* metdatadb_env +* metplotpy_env +* netcdf4_env +* pygrib_env +* spacetime_env +* weatherregime_env +* xesmf_env + +Example:: + + spacetime_env + +The above example uses the Conda environment +in dtcenter/metplus-envs:**spacetime** to run a user script. +Note that only one dependency that contains the **_env** suffix can be supplied +to a given use case. + +Other Environments +"""""""""""""""""" + +A few of the environments do not contain Conda environments and +are handled a little differently. + +* **gempak_env** - Used if GempakToCF.jar is needed for a use case to convert + GEMPAK data to NetCDF format so it can be read by the MET tools. + Instead of creating a Python environment to use for the use case, + this Docker image installs Java and obtains the GempakToCF.jar file. + When creating the Docker container to run the use cases, + the necessary Java files are copied over into the container + that runs the use cases so that the JAR file can be run by METplus wrappers. +* **gfdl-tracker_env** - Contains the GFDL Tracker application that is used by + the GFDLTracker wrapper use cases. + + +Other Keywords +"""""""""""""" + +Besides specifying Python environments, +there are additional keywords that can be used to set up the environment +to run a use case: + +* **py_embed** - Used if a different Python environment is required to + run a Python Embedding script. If this keyword is included with a Python + environment, then the MET_PYTHON_EXE environment variable will be set to + specify the version of Python3 that is included in that environment + +Example:: + + pygrib_env,py_embed + +In this example, the dtcenter/metplus-envs:**pygrib** environment is used to +run the use case. Since **py_embed** is also included, then the following will +be added to the call to run_metplus.py so that the Python embedding script +will use the **pygrib** environment to run:: + + user_env_vars.MET_PYTHON_EXE=/usr/local/envs/pygrib/bin/python3 + +Please see the +`MET User's Guide `_ +for more information on how to use Python Embedding. + +* **metviewer** - Used if METviewer should be made available to the use case. + This is typically added for a METdbLoad use case that needs to populate a + database with MET output. + +* **metplus** - Used if a user script needs to call utility functions from the + metplus Python package. This keyword simply adds the METplus source code + directory to the PYTHONPATH so that the metplus.util functions can be + imported. Note that this keyword is not needed unless a different Python + environment is specified with a "_env" keyword. The version of Python that + is used to run typical use cases has already installed the METplus Python + package in its environment, so the package can be imported easily. + +* **metdatadb** - Used if the METdatadb repository is needed to run. Note that + this is only needed if using a Conda environment other than metdatadb_env. + The repository Python code will be installed in the Python environment. + +* **cartopy** - Used if cartopy 0.18.0 is needed in the Conda environment. + Cartopy uses shapefiles that are downloaded as needed. The URL that is used + to download the files has changed since cartopy 0.18.0 and we have run into + issues where the files cannot be obtained. To remedy this issue, we modified + the scripts that generate the Docker images that contain the Conda + environments that use cartopy to download these shape files so they will + always be available. These files need to be copied from the Docker + environment image into the testing image. When this keyword is found in the + dependency list, a different Dockerfile (Dockerfile.run_cartopy found in + .github/actions/run_tests) is used to create the testing environment and + copy the required shapefiles into place. + + +Creating New Python Environments +"""""""""""""""""""""""""""""""" + +In METplus v4.0.0 and earlier, a list of Python packages were added to use +cases that required additional packages. These packages were either installed +with pip3 or using a script. This approach was very time consuming as some +packages take a very long time to install in Docker. The new approach involves +creating Docker images that use Conda to create a Python environment that can +run the use case. To see what is available in each of the existing Python +environments, refer to the comments in the scripts found in +**scripts/docker/docker_env/scripts**. +New environments must be added by a METplus developer, +so please create a discussion on the +`METplus GitHub Discussions `_ +forum if none of these environments contain the package requirements +needed to run a new use case. + +A **README.md** file can be found in **scripts/docker/docker_env** that +provides commands that can be run to recreate a Docker image if the +conda environment needs to be updated. Please note that Docker must +be installed on the workstation used to create new Docker images and +a DockerHub account with access to the dtcenter repositories must +be used to push Docker images to DockerHub. + +The **README.md** file also contains commands to create a conda environment +that is used for the tests locally. Any base conda environments, +such as metplus_base and py_embed_base, must be created locally first +before creating an environment that builds upon these environments. +Please note that some commands in the scripts are specific to +the Docker environment and may need to be rerun to successfully +build the environment locally. + +**Installing METplus Components** + +The scripts used to create the Python environment Docker images +do not install any METplus components, +such as METplotpy, METcalcpy, METdatadb, and METplus, +in the Python environment that may be needed for a use case. +This is done because the automated tests +will install and use the latest version (develop) of the packages to +ensure that any changes to those components do not break any existing +use cases. These packages will need to be installed by the user +and need to be updated manually. To install these packages, +activate the Conda environment, obtain the source code from GitHub, +and run "pip3 install ." in the top level directory of the repository. + +Example:: + + conda activate weatherregime + git clone git@github.com:dtcenter/METplotpy + cd METplotpy + git checkout develop + git pull + pip3 install . + +**Cartopy Shapefiles** + +The cartopy python package automatically attempts to download +shapefiles as needed. +The URL that is used in cartopy version 0.18.0 and earlier no longer +exists, so use cases that needs these files will fail if they are +not found locally. If a conda environment uses cartopy, these +shapefiles may need to be downloaded by the user running the use case +even if the conda environment was created by another user. +Cartopy provides a script that can be used to obtain these shapefiles +from the updated URL:: + + wget https://raw.githubusercontent.com/SciTools/cartopy/master/tools/cartopy_feature_download.py + python3 cartopy_feature_download.py cultural physical cultural-extra + + +.. _cg-ci-use-case-groups: + +Use Case Groups +^^^^^^^^^^^^^^^ + +The use cases that are run in the automated test suite are divided into +groups that can be run concurrently. + +The **use_case_groups.json** file (found in **.github/parm**) +contains a list of the use case groups to run together. +In METplus version 4.0.0 and earlier, this list was +found in the .github/workflows/testing.yml file. + +Each use case group is defined with the following format:: + + { + "category": "", + "index_list": "", + "run": + } + +* **** is the category group that the use case is found under in the + **all_use_cases.txt** file (see :ref:`cg-ci-all-use-cases`). +* **** is a list of indices of the use cases from + **all_use_cases.txt** to run in the group. + This can be a single integer, a comma-separated list of + integers, and a range of values with a dash, i.e. 0-3. +* **** is a boolean (true/false) value that determines if the use + case group should be run. If the workflow job controls are not set to run + all of the use cases, then only use case groups that are set to true are + run. + +Example:: + + { + "category": "climate", + "index_list": "2", + "run": true + } + +This example defines a use case group that contains the climate use case +with index 2 and is marked to run for every push. + + +.. _cg-ci-subset_category: + +Subset Category into Multiple Tests +""""""""""""""""""""""""""""""""""" + +Use cases can be separated into multiple test jobs. +In the *index_list* value, define the cases to run for the job. +Use cases are numbered starting with 0 and correspond to the number set in +the **all_use_cases.txt** file. + +The argument supports a comma-separated list of numbers. Example:: + + { + "category": "data_assimilation", + "index_list": "0,2,4", + "run": false + }, + { + "category": "data_assimilation", + "index_list": "1,3", + "run": false + }, + +The above example will run a job with data_assimilation use cases 0, 2, and +4, then another job with data_assimilation use cases 1 and 3. + +It also supports a range of numbers separated with a dash. Example:: + + { + "category": "data_assimilation", + "index_list": "0-3", + "run": false + }, + { + "category": "data_assimilation", + "index_list": "4-5", + "run": false + }, + +The above example will run a job with data_assimilation 0, 1, 2, and 3, then +another job with data_assimilation 4 and 5. + +You can also use a combination of commas and dashes to define the list of cases +to run. Example:: + + { + "category": "data_assimilation", + "index_list": "0-2,4", + "run": false + }, + { + "category": "data_assimilation", + "index_list": "3", + "run": false + }, + +The above example will run data_assimilation 0, 1, 2, and 4 in one +job, then data_assimilation 3 in another job. + +Run Use Cases +^^^^^^^^^^^^^ + +The **use_case_tests** job is duplicated for each use case group using the +strategy -> matrix syntax:: + + strategy: + fail-fast: false + matrix: ${{fromJson(needs.job_control.outputs.matrix)}} + +**fail-fast** is set to false so that the rest of the use case test jobs will +run even when one of them fails. The **matrix** value is a list of use +case categories and indices that is created in the :ref:`cg-ci-job-control` +job. Each value in the list is referenced in the job steps with +**${{ matrix.categories }}**:: + + - name: Run Use Cases + uses: ./.github/actions/run_tests + id: run_tests + with: + categories: ${{ matrix.categories }} + +The logic that runs the use cases is contained in a custom GitHub Action +that is found in the METplus repository. + +Obtaining Input Data +"""""""""""""""""""" + +Each use case category has a corresponding Docker data volume that contains +the input data needed to run all of the use cases. The data volume is obtained +from DockerHub and mounted into the container that will run the use cases +using the **\-\-volumes-from** argument to the **docker run** command. + +Build Docker Test Environment +""""""""""""""""""""""""""""" + +A `Docker multi-stage build `_ +is used to create the Docker environment to run the use cases. +The Docker images that contain the :ref:`cg-ci-use-case-dependencies` are +built and the relevant files (such as the Conda environment files) are +copied into the METplus image so that they will be available when running +the use cases. + +Setup Use Case Commands +""""""""""""""""""""""" + +Before **run_metplus.py** is called to run the use case, +some other commands are run in the Docker container. +For example, if another METplus Python component such as +METcalcpy, METplotpy, or METdatadb are required for the use case, +the **develop** branch of those repositories are obtained the Python code +is installed in the Python (Conda) environment that will be used to +run the use case. + +Run the Use Cases +""""""""""""""""" + +The **run_metplus.py** script is called to run each use case. +The **OUTPUT_BASE** METplus configuration variable is overridden to +include the use case name identifier defined in +the :ref:`cg-ci-all-use-cases` file to isolate all of the output for each +use case. If any of the use cases contain an error, then the job for the +use case group will fail and display a red X next to the job on the +GitHub Actions webpage. + +Difference Tests +^^^^^^^^^^^^^^^^ + +After all of the use cases in a group have finished running, the output +that was generated is compared to the truth data to determine if any of +the output was changed. The truth data for each use case group is stored +in a Docker data volume on DockerHub. The **diff_util.py** script +(found in **metplus/util**) is run to compare all of the output files in +different ways depending on the file type. + +The logic in this script could be improved to provide more robust testing. +For example, the logic to compare images has been disabled because the +existing logic was reporting false differences. + +If any differences were found, then the files that contained the differences +are copied into a directory so they can be made available in an artifact. +The files are renamed to include an identifier just before the extension +so that it is easy to tell which file came from the truth data and which came +from the new output. + +.. _cg-ci-create-output-data-volumes: + +Create/Update Output Data Volumes +--------------------------------- + +Differences in the use case output may be expected. +The most common difference is new data from a newly added use case that is +not found in the truth data. If all of the differences are determined to be +expected, then the truth data must be updated so that the changes are included +in future difference tests. +All of the artifacts with a name that starts with **use_cases_** are downloaded +in this job. Data from each group is copied into a Docker image and pushed +up to DockerHub, replacing the images that were used for the difference tests. +See :ref:`cg-ci-push-reference-branch` for information on which events +trigger this job. + +Output (Artifacts) +------------------ + +Error Logs +^^^^^^^^^^ + +If there are errors in any of the use cases, then the log file from the run +is copied into a directory that will be made available at the end of the +workflow run as a downloadable artifact. This makes it easier to review all +of the log files that contain errors. + +Output Data +^^^^^^^^^^^ + +All of the output data that is generated by the use case groups are saved as +downloadable artifacts. Each output artifact name starts with **use_cases_** +and contains the use case category and indices. This makes it easy to obtain +the output from a given use case to review. + +Diff Data +^^^^^^^^^ + +When differences are found when comparing the new output from a use case to +the truth data, an artifact is created for the use case group. It contains +files that differ so that the user can download and examine them. Files that +are only found in one or the other are also included. diff --git a/docs/Contributors_Guide/figure/ci-doc-artifacts.png b/docs/Contributors_Guide/figure/ci-doc-artifacts.png new file mode 100644 index 0000000000..37ac0b6f35 Binary files /dev/null and b/docs/Contributors_Guide/figure/ci-doc-artifacts.png differ diff --git a/docs/Contributors_Guide/figure/ci-doc-error.png b/docs/Contributors_Guide/figure/ci-doc-error.png new file mode 100644 index 0000000000..6e9a149f36 Binary files /dev/null and b/docs/Contributors_Guide/figure/ci-doc-error.png differ diff --git a/docs/Contributors_Guide/figure/gha-workflow-name.png b/docs/Contributors_Guide/figure/gha-workflow-name.png old mode 100755 new mode 100644 index bfcd5cb8f3..4d62965c7a Binary files a/docs/Contributors_Guide/figure/gha-workflow-name.png and b/docs/Contributors_Guide/figure/gha-workflow-name.png differ diff --git a/docs/Users_Guide/installation.rst b/docs/Users_Guide/installation.rst index be724cc5a3..ab33891e3d 100644 --- a/docs/Users_Guide/installation.rst +++ b/docs/Users_Guide/installation.rst @@ -202,7 +202,6 @@ The METplus Wrappers source code contains the following directory structure:: METplus/ build_components/ - ci/ docs/ environment.yml internal_tests/ @@ -212,6 +211,7 @@ The METplus Wrappers source code contains the following directory structure:: produtil/ README.md requirements.txt + scripts/ setup.py ush/ @@ -222,9 +222,6 @@ The **build_components/** directory contains scripts that use manage_externals and files available on dtcenter.org to download MET and start the build process. -The **ci/** directory contains scripts that are used for creating -Docker images and scripts that are used internally for automation. - The **docs/** directory contains documentation for users and contributors (HTML) and Doxygen files that are used to create the METplus wrapper API documentation. The @@ -247,6 +244,9 @@ METplus Wrappers. The **produtil/** directory contains part of the external utility produtil. +The **scripts/** directory contains scripts that are used for creating +Docker images. + The **ush/** directory contains the run_metplus.py script that is executed to run use cases. diff --git a/internal_tests/use_cases/all_use_cases.txt b/internal_tests/use_cases/all_use_cases.txt index fc4c3c0cd5..051b1e961e 100644 --- a/internal_tests/use_cases/all_use_cases.txt +++ b/internal_tests/use_cases/all_use_cases.txt @@ -1,5 +1,5 @@ Category: met_tool_wrapper -0::CyclonePlotter::met_tool_wrapper/CyclonePlotter/CyclonePlotter.conf,user_env_vars.MET_PYTHON_EXE=python3:: cycloneplotter_env,cartopy +0::CyclonePlotter::met_tool_wrapper/CyclonePlotter/CyclonePlotter.conf:: cycloneplotter_env,cartopy, py_embed 1::PCPCombine_python_embedding:: met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf:: h5py_env,py_embed 2::ASCII2NC:: met_tool_wrapper/ASCII2NC/ASCII2NC.conf 3::met_tool_wrapper/ASCII2NC/ASCII2NC_python_embedding.conf @@ -130,7 +130,7 @@ Category: s2s 9:: UserScript_obsERA_obsOnly_OMI:: model_applications/s2s/UserScript_obsERA_obsOnly_OMI.conf:: spacetime_env, metdatadb 10:: UserScript_obsERA_obsOnly_RMM:: model_applications/s2s/UserScript_obsERA_obsOnly_RMM.conf:: spacetime_env, metdatadb 11:: UserScript_fcstGFS_obsERA_WeatherRegime:: model_applications/s2s/UserScript_fcstGFS_obsERA_WeatherRegime.conf:: weatherregime_env,cartopy,metplus -12:: UserScript_obsERA_obsOnly_Stratosphere:: model_applications/s2s/UserScript_obsERA_obsOnly_Stratosphere.conf:: metplotpy_env,metdatadb,metplus +12:: UserScript_obsERA_obsOnly_Stratosphere:: model_applications/s2s/UserScript_obsERA_obsOnly_Stratosphere.conf:: metplotpy_env,metdatadb Category: space_weather 0::GridStat_fcstGloTEC_obsGloTEC_vx7:: model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf diff --git a/ci/docker/Dockerfile b/scripts/docker/Dockerfile similarity index 100% rename from ci/docker/Dockerfile rename to scripts/docker/Dockerfile diff --git a/ci/docker/docker_data/Dockerfile b/scripts/docker/docker_data/Dockerfile similarity index 100% rename from ci/docker/docker_data/Dockerfile rename to scripts/docker/docker_data/Dockerfile diff --git a/ci/docker/docker_data/build_docker_images.sh b/scripts/docker/docker_data/build_docker_images.sh similarity index 100% rename from ci/docker/docker_data/build_docker_images.sh rename to scripts/docker/docker_data/build_docker_images.sh diff --git a/ci/docker/docker_data/hooks/build b/scripts/docker/docker_data/hooks/build similarity index 100% rename from ci/docker/docker_data/hooks/build rename to scripts/docker/docker_data/hooks/build diff --git a/ci/docker/docker_data_output/Dockerfile b/scripts/docker/docker_data_output/Dockerfile similarity index 100% rename from ci/docker/docker_data_output/Dockerfile rename to scripts/docker/docker_data_output/Dockerfile diff --git a/ci/docker/docker_env/Dockerfile b/scripts/docker/docker_env/Dockerfile similarity index 100% rename from ci/docker/docker_env/Dockerfile rename to scripts/docker/docker_env/Dockerfile diff --git a/ci/docker/docker_env/Dockerfile.gempak_env b/scripts/docker/docker_env/Dockerfile.gempak_env similarity index 100% rename from ci/docker/docker_env/Dockerfile.gempak_env rename to scripts/docker/docker_env/Dockerfile.gempak_env diff --git a/ci/docker/docker_env/Dockerfile.gfdl-tracker b/scripts/docker/docker_env/Dockerfile.gfdl-tracker similarity index 100% rename from ci/docker/docker_env/Dockerfile.gfdl-tracker rename to scripts/docker/docker_env/Dockerfile.gfdl-tracker diff --git a/ci/docker/docker_env/Dockerfile.metplus_base b/scripts/docker/docker_env/Dockerfile.metplus_base similarity index 100% rename from ci/docker/docker_env/Dockerfile.metplus_base rename to scripts/docker/docker_env/Dockerfile.metplus_base diff --git a/ci/docker/docker_env/Dockerfile.py_embed_base b/scripts/docker/docker_env/Dockerfile.py_embed_base similarity index 100% rename from ci/docker/docker_env/Dockerfile.py_embed_base rename to scripts/docker/docker_env/Dockerfile.py_embed_base diff --git a/ci/docker/docker_env/README.md b/scripts/docker/docker_env/README.md similarity index 99% rename from ci/docker/docker_env/README.md rename to scripts/docker/docker_env/README.md index dc40f5ad0c..9b8adc0a63 100644 --- a/ci/docker/docker_env/README.md +++ b/scripts/docker/docker_env/README.md @@ -1,6 +1,6 @@ # Docker Conda Environments -Run the commands from this directory (ci/docker/docker_env). +Run the commands from this directory (scripts/docker/docker_env). Instructions include how to create Docker images in dtcenter/metplus-envs so environments are available for the automated tests. Instructions to create these Conda environments on a local machine are also provided. diff --git a/ci/docker/docker_env/scripts/cfgrib_env.sh b/scripts/docker/docker_env/scripts/cfgrib_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/cfgrib_env.sh rename to scripts/docker/docker_env/scripts/cfgrib_env.sh diff --git a/ci/docker/docker_env/scripts/cycloneplotter_env.sh b/scripts/docker/docker_env/scripts/cycloneplotter_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/cycloneplotter_env.sh rename to scripts/docker/docker_env/scripts/cycloneplotter_env.sh diff --git a/ci/docker/docker_env/scripts/diff_env.sh b/scripts/docker/docker_env/scripts/diff_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/diff_env.sh rename to scripts/docker/docker_env/scripts/diff_env.sh diff --git a/ci/docker/docker_env/scripts/gempak_env.sh b/scripts/docker/docker_env/scripts/gempak_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/gempak_env.sh rename to scripts/docker/docker_env/scripts/gempak_env.sh diff --git a/ci/docker/docker_env/scripts/h5py_env.sh b/scripts/docker/docker_env/scripts/h5py_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/h5py_env.sh rename to scripts/docker/docker_env/scripts/h5py_env.sh diff --git a/ci/docker/docker_env/scripts/icecover_env.sh b/scripts/docker/docker_env/scripts/icecover_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/icecover_env.sh rename to scripts/docker/docker_env/scripts/icecover_env.sh diff --git a/ci/docker/docker_env/scripts/metdatadb_env.sh b/scripts/docker/docker_env/scripts/metdatadb_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/metdatadb_env.sh rename to scripts/docker/docker_env/scripts/metdatadb_env.sh diff --git a/ci/docker/docker_env/scripts/metplotpy_env.sh b/scripts/docker/docker_env/scripts/metplotpy_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/metplotpy_env.sh rename to scripts/docker/docker_env/scripts/metplotpy_env.sh diff --git a/ci/docker/docker_env/scripts/metplus_base_env.sh b/scripts/docker/docker_env/scripts/metplus_base_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/metplus_base_env.sh rename to scripts/docker/docker_env/scripts/metplus_base_env.sh diff --git a/ci/docker/docker_env/scripts/netcdf4_env.sh b/scripts/docker/docker_env/scripts/netcdf4_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/netcdf4_env.sh rename to scripts/docker/docker_env/scripts/netcdf4_env.sh diff --git a/ci/docker/docker_env/scripts/py_embed_base_env.sh b/scripts/docker/docker_env/scripts/py_embed_base_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/py_embed_base_env.sh rename to scripts/docker/docker_env/scripts/py_embed_base_env.sh diff --git a/ci/docker/docker_env/scripts/pygrib_env.sh b/scripts/docker/docker_env/scripts/pygrib_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/pygrib_env.sh rename to scripts/docker/docker_env/scripts/pygrib_env.sh diff --git a/ci/docker/docker_env/scripts/pytest_env.sh b/scripts/docker/docker_env/scripts/pytest_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/pytest_env.sh rename to scripts/docker/docker_env/scripts/pytest_env.sh diff --git a/ci/docker/docker_env/scripts/spacetime_env.sh b/scripts/docker/docker_env/scripts/spacetime_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/spacetime_env.sh rename to scripts/docker/docker_env/scripts/spacetime_env.sh diff --git a/ci/docker/docker_env/scripts/weatherregime_env.sh b/scripts/docker/docker_env/scripts/weatherregime_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/weatherregime_env.sh rename to scripts/docker/docker_env/scripts/weatherregime_env.sh diff --git a/ci/docker/docker_env/scripts/xesmf_env.sh b/scripts/docker/docker_env/scripts/xesmf_env.sh similarity index 100% rename from ci/docker/docker_env/scripts/xesmf_env.sh rename to scripts/docker/docker_env/scripts/xesmf_env.sh diff --git a/ci/docker/hooks/build b/scripts/docker/hooks/build similarity index 100% rename from ci/docker/hooks/build rename to scripts/docker/hooks/build diff --git a/ci/docker/hooks/get_met_version b/scripts/docker/hooks/get_met_version similarity index 100% rename from ci/docker/hooks/get_met_version rename to scripts/docker/hooks/get_met_version