wrf-model
diff --git a/.ci/azure-pipelines/matrix.yml b/.ci/azure-pipelines/matrix.yml
@@ -0,0 +1,103 @@
+# WRF-CMake (https://github.com/WRF-CMake/wrf).
+# Copyright 2019 M. Riechert and D. Meyer. Licensed under the MIT License.
+
+parameters:
+  templates:
+    Ubuntu: unix.yml
+    CentOS: unix.yml
+    macOS: unix.yml
+    Windows: windows.yml
+
+  Ubuntu:
+    CC: gcc-11
+    FC: gfortran-11
+  CentOS:
+    CC: gcc
+    FC: gfortran
+  macOS:
+    CC: gcc-11
+    FC: gfortran-11
+  Windows:
+    CC: gcc
+    FC: gfortran
+
+  common:
+    WATS_REPO: WRF-CMake/wats
+    WATS_BRANCH: master
+    WATS_MODE: wrf
+    WATS_WPS_PKG_VERSION: 0.0.2
+    NESTING: basic
+    GRIB1: 0
+    GRIB2: 1
+    REAL8: 0
+
+  RELEASE: false
+
+  # Loop/matrix parameters.
+  # Note that these may be overridden by the including yml (typically azure-pipelines.yml).
+  #OS_NAMES: [Windows, Ubuntu, CentOS, macOS]
+  # macOS temporarily disabled due to segfaults.
+  OS_NAMES: [Windows, Ubuntu, CentOS]
+  BUILD_SYSTEMS: [Make, CMake]
+  BUILD_TYPES: [Debug, Release]
+  MODES: [serial, smpar, dmpar, dm_sm]
+
+jobs:
+- ${{ each OS_NAME in parameters.OS_NAMES }}:
+  - ${{ each BUILD_TYPE in parameters.BUILD_TYPES }}:
+    - ${{ each MODE in parameters.MODES }}:
+
+      - ${{ each BUILD_SYSTEM in parameters.BUILD_SYSTEMS }}:
+        # For Windows, since there is no Make-based build, skip that combination.
+        - ${{ if not(and(eq(OS_NAME, 'Windows'), eq(BUILD_SYSTEM, 'Make'))) }}:
+          - template: ${{ parameters.templates[OS_NAME] }}
+            parameters:
+              RELEASE: ${{ parameters.RELEASE }}
+              OS_NAME: ${{ OS_NAME }}
+              BUILD_SYSTEM: ${{ BUILD_SYSTEM }}
+              BUILD_TYPE: ${{ BUILD_TYPE }}
+              MODE: ${{ MODE }}
+              ${{ insert }}: ${{ parameters[OS_NAME] }}
+              ${{ insert }}: ${{ parameters.common }}
+              ${{ if or(eq(parameters.WATS_PLOTS, 'true'), eq(parameters.WATS_DIFF, 'true')) }}:
+                WATS_RUN: true
+
+      # Make vs CMake comparison
+      - ${{ if eq(parameters.WATS_DIFF, 'true') }}:
+        - template: wats_diff.yml
+          parameters:
+            # For Windows, since there is no Make-based reference, compare against Ubuntu.
+            ${{ if eq(OS_NAME, 'Windows') }}:
+              OS_NAME_LEFT: Ubuntu
+            ${{ if ne(OS_NAME, 'Windows') }}:
+              OS_NAME_LEFT: ${{ OS_NAME }}
+            OS_NAME_RIGHT: ${{ OS_NAME }}
+            BUILD_SYSTEM_LEFT: Make
+            BUILD_SYSTEM_RIGHT: CMake
+            BUILD_TYPE: ${{ BUILD_TYPE }}
+            MODE: ${{ MODE }}
+            ${{ insert }}: ${{ parameters.common }}
+
+# Make vs Make comparison
+- ${{ if eq(parameters.WATS_DIFF, 'true') }}:
+  - ${{ each BUILD_TYPE in parameters.BUILD_TYPES }}:
+    - ${{ each MODE in parameters.MODES }}:
+      - template: wats_diff.yml
+        parameters:
+          OS_NAME_LEFT: Ubuntu
+          OS_NAME_RIGHT: macOS
+          BUILD_SYSTEM_LEFT: Make
+          BUILD_SYSTEM_RIGHT: Make
+          BUILD_TYPE: ${{ BUILD_TYPE }}
+          MODE: ${{ MODE }}
+          ${{ insert }}: ${{ parameters.common }}
+
+- ${{ if eq(parameters.WATS_PLOTS, 'true') }}:
+  - template: wats_plots.yml
+    parameters:
+      OS_NAMES: ${{ parameters.OS_NAMES }}
+      BUILD_SYSTEMS: ${{ parameters.BUILD_SYSTEMS }}
+      BUILD_TYPES: ${{ parameters.BUILD_TYPES }}
+      MODES: ${{ parameters.MODES }}
+      vars:
+        ${{ insert }}: ${{ parameters.common }}
diff --git a/.ci/azure-pipelines/release.yml b/.ci/azure-pipelines/release.yml
@@ -0,0 +1,55 @@
+steps:
+- ${{ if eq(parameters.OS_NAME, 'Windows') }}:
+  - task: ArchiveFiles@2
+    inputs:
+      rootFolderOrFile: build/install
+      includeRootFolder: false
+      archiveType: zip
+      archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(NESTING)_nesting-$(MODE)-x64-$(OS_NAME)-$(BUILD_TYPE).zip' 
+    displayName: Create distribution package
+
+- ${{ if not(eq(parameters.OS_NAME, 'Windows')) }}:
+  - bash: $(run) .ci/unix/use-conda.sh
+    displayName: Enable Conda
+
+  - bash: $(run) .ci/unix/delocate.sh
+    displayName: Delocate
+
+  - task: ArchiveFiles@2
+    inputs:
+      rootFolderOrFile: build/install
+      includeRootFolder: false
+      archiveType: tar
+      tarCompression: xz
+      archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(NESTING)_nesting-$(MODE)-x64-$(OS_NAME)-$(BUILD_TYPE).tar.xz' 
+    displayName: Create distribution package
+
+- bash: |
+    set -ex
+    cd "$(Build.ArtifactStagingDirectory)"
+    # CentOS packages will run on most glibc-based Linux distributions
+    find . -name '*CentOS*' -exec bash -c 'mv $0 ${0/CentOS/Linux}' {} \;
+    # Rename all files to lowercase
+    for file in $(ls); do mv ${file} $(echo ${file} | tr '[:upper:]' '[:lower:]'); done
+  displayName: Rename package files
+
+# not strictly needed
+- task: PublishBuildArtifacts@1
+  displayName: Store distribution package
+  inputs:
+    pathtoPublish: $(Build.ArtifactStagingDirectory)
+    artifactName: dist_$(OS_NAME)_$(BUILD_SYSTEM)_$(BUILD_TYPE)_$(MODE)
+
+- task: GithubRelease@0 
+  displayName: Create GitHub Draft Release
+  inputs:
+    gitHubConnection: WRF-CMake-releases
+    repositoryName: WRF-CMake/wrf
+    action: edit # will create if not existing
+    tag: $(Build.SourceBranchName)
+    isDraft: true
+    isPreRelease: true
+    addChangeLog: false
+    assetUploadMode: replace
+    assets: |
+      $(Build.ArtifactStagingDirectory)/*
diff --git a/.ci/azure-pipelines/unix.yml b/.ci/azure-pipelines/unix.yml
@@ -0,0 +1,49 @@
+# WRF-CMake (https://github.com/WRF-CMake/wrf).
+# Copyright 2019 M. Riechert and D. Meyer. Licensed under the MIT License.
+
+jobs:
+- job: ${{ parameters.OS_NAME }}_${{ parameters.BUILD_SYSTEM }}_${{ parameters.BUILD_TYPE }}_${{ parameters.MODE }}
+  displayName: '${{ parameters.OS_NAME }} ${{ parameters.BUILD_SYSTEM }}: ${{ parameters.BUILD_TYPE }}, ${{ parameters.MODE }}'
+
+  timeoutInMinutes: 0
+
+  pool:
+    ${{ if or(eq(parameters.OS_NAME, 'Ubuntu'), eq(parameters.OS_NAME, 'CentOS')) }}:
+      vmImage: ubuntu-latest
+    ${{ if eq(parameters.OS_NAME, 'macOS') }}:
+      vmImage: macOS-11
+
+  variables:
+    ${{ insert }}: ${{ parameters }}
+    ${{ if eq(parameters.OS_NAME, 'Ubuntu') }}:
+      IMAGE: ubuntu:20.04
+      run: .ci/unix/run-in-docker.sh
+    ${{ if eq(parameters.OS_NAME, 'CentOS') }}:
+      IMAGE: quay.io/pypa/manylinux2014_x86_64:latest
+      run: .ci/unix/run-in-docker.sh
+    ${{ if eq(parameters.OS_NAME, 'macOS') }}:
+      run: ''
+
+  steps:
+  - script: .ci/unix/increase-swap-space.sh
+    displayName: Increase swap space
+
+  - script: $(run) .ci/unix/dump-vm-specs.sh
+    displayName: Dump VM specs
+
+  - script: $(run) printenv | sort
+    displayName: Dump environment variables
+
+  - script: $(run) .ci/unix/setup-dependencies.sh
+    displayName: Setup dependencies
+
+  - script: $(run) .ci/unix/install-wrf.sh
+    displayName: Install WRF
+
+  - ${{ if eq(parameters.WATS_RUN, 'true') }}:
+    - template: wats_run.yml
+
+  - ${{ if eq(parameters.RELEASE, 'true') }}:
+    - template: release.yml
+      parameters:
+        ${{ insert }}: ${{ parameters }}
diff --git a/.ci/azure-pipelines/wats_diff.yml b/.ci/azure-pipelines/wats_diff.yml
@@ -0,0 +1,50 @@
+# WRF-CMake (https://github.com/WRF-CMake/wrf).
+# Copyright 2019 M. Riechert and D. Meyer. Licensed under the MIT License.
+
+jobs:
+- job: wats_diff_${{ parameters.OS_NAME_LEFT }}_${{ parameters.OS_NAME_RIGHT }}_${{ parameters.BUILD_SYSTEM_LEFT }}_${{ parameters.BUILD_SYSTEM_RIGHT }}_${{ parameters.BUILD_TYPE }}_${{ parameters.MODE }}
+  displayName: 'WATS ${{ parameters.OS_NAME_LEFT }}/${{ parameters.OS_NAME_RIGHT }} ${{ parameters.BUILD_SYSTEM_LEFT }}/${{ parameters.BUILD_SYSTEM_RIGHT }}: ${{ parameters.BUILD_TYPE }}, ${{ parameters.MODE }}'
+
+  pool:
+    vmImage: ubuntu-latest
+
+  dependsOn:
+  - ${{ parameters.OS_NAME_LEFT }}_${{ parameters.BUILD_SYSTEM_LEFT }}_${{ parameters.BUILD_TYPE }}_${{ parameters.MODE }}
+  - ${{ parameters.OS_NAME_RIGHT }}_${{ parameters.BUILD_SYSTEM_RIGHT }}_${{ parameters.BUILD_TYPE }}_${{ parameters.MODE }}
+
+  variables:
+    ${{ insert }}: ${{ parameters }}
+
+  steps:
+  - bash: .ci/unix/use-conda.sh
+    displayName: Enable Conda
+
+  - bash: .ci/unix/install-wats.sh
+    displayName: Install WATS
+
+  - task: DownloadBuildArtifacts@0
+    displayName: Retrieve ${{ parameters.OS_NAME_LEFT }} ${{ parameters.BUILD_SYSTEM_LEFT }} WATS outputs
+    inputs:
+      downloadPath: wats_outputs
+      downloadType: specific
+      itemPattern: 'wats_${{ parameters.OS_NAME_LEFT }}_${{ parameters.BUILD_SYSTEM_LEFT }}_${{ parameters.BUILD_TYPE }}_${{ parameters.MODE }}/**'
+
+  - task: DownloadBuildArtifacts@0
+    displayName: Retrieve ${{ parameters.OS_NAME_RIGHT }} ${{ parameters.BUILD_SYSTEM_RIGHT }} WATS outputs
+    inputs:
+      downloadPath: wats_outputs
+      downloadType: specific
+      itemPattern: 'wats_${{ parameters.OS_NAME_RIGHT }}_${{ parameters.BUILD_SYSTEM_RIGHT }}_${{ parameters.BUILD_TYPE }}_${{ parameters.MODE }}/**'
+
+  - script: |
+      cd wats_outputs
+      find . -type f
+    displayName: List WATS output files
+  
+  - script: |
+      set -ex
+      O=wats_outputs
+      W="python wats/wats/main.py diff --mode $WATS_MODE"
+      $W $O/wats_${OS_NAME_LEFT}_${BUILD_SYSTEM_LEFT}_${BUILD_TYPE}_${MODE} $O/wats_${OS_NAME_RIGHT}_${BUILD_SYSTEM_RIGHT}_${BUILD_TYPE}_${MODE}
+    displayName: Run WATS diff
+      
diff --git a/.ci/azure-pipelines/wats_plots.yml b/.ci/azure-pipelines/wats_plots.yml
@@ -0,0 +1,125 @@
+# WRF-CMake (https://github.com/WRF-CMake/wrf).
+# Copyright 2019 M. Riechert and D. Meyer. Licensed under the MIT License.
+
+parameters:
+  DOMAINS: [d01, d02]
+  TIMES: [0, 6]
+
+jobs:
+- job: wats_plots
+  displayName: 'WATS plots'
+
+  timeoutInMinutes: 0
+
+  pool:
+    vmImage: ubuntu-latest
+
+  dependsOn:
+  - ${{ each OS_NAME in parameters.OS_NAMES }}:
+    - ${{ each BUILD_TYPE in parameters.BUILD_TYPES }}:
+      - ${{ each MODE in parameters.MODES }}:
+        - ${{ each BUILD_SYSTEM in parameters.BUILD_SYSTEMS }}:
+          # For Windows, since there is no Make-based build, skip that combination.
+          - ${{ if not(and(eq(OS_NAME, 'Windows'), eq(BUILD_SYSTEM, 'Make'))) }}:
+            - ${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }}
+
+  variables:
+    ${{ insert }}: ${{ parameters.vars }}
+
+  steps:
+  - bash: .ci/unix/use-conda.sh
+    displayName: Enable Conda
+
+  - bash: .ci/unix/install-wats.sh
+    displayName: Install WATS
+
+  # Plot: Ubuntu/Make/Debug/serial against all others
+
+  - task: DownloadBuildArtifacts@0
+    displayName: Retrieve reference Ubuntu/Make/Debug/serial WATS outputs
+    inputs:
+      downloadPath: wats_outputs
+      downloadType: specific
+      itemPattern: 'wats_Ubuntu_Make_Debug_serial/**'
+
+  - ${{ each OS_NAME in parameters.OS_NAMES }}:
+    - ${{ each BUILD_TYPE in parameters.BUILD_TYPES }}:
+      - ${{ each MODE in parameters.MODES }}:
+        - ${{ each BUILD_SYSTEM in parameters.BUILD_SYSTEMS }}:
+          # For Windows, since there is no Make-based build, skip that combination.
+          # Also, skip the reference combination.
+          - ${{ if not(or(and(eq(OS_NAME, 'Windows'), eq(BUILD_SYSTEM, 'Make')), and(eq(OS_NAME, 'Ubuntu'), eq(BUILD_SYSTEM, 'Make'), eq(MODE, 'serial'), eq(BUILD_TYPE, 'Debug')))) }}:
+
+            - task: DownloadBuildArtifacts@0
+              displayName: Retrieve trial ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} WATS outputs
+              inputs:
+                downloadPath: wats_outputs
+                downloadType: specific
+                itemPattern: 'wats_${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }}/**'
+
+            - ${{ each DOMAIN in parameters.DOMAINS }}:
+              - ${{ each TIME in parameters.TIMES }}:
+                - script: |
+                    set -ex
+                    O=wats_outputs
+                    W="python wats/wats/plots.py compute --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --filter ${{ DOMAIN }} --time-idx ${{ TIME }} --append"
+                    $W $O/wats_Ubuntu_Make_Debug_serial $O/wats_${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }}
+                  displayName: 'Run WATS: compute plot stats Ubuntu/Make/Debug/serial vs ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} for domain ${{ DOMAIN }} - T${{ TIME }}'
+                
+            - script: |
+                rm -rf wats_outputs/wats_${{ OS_NAME }}_${{ BUILD_SYSTEM }}_${{ BUILD_TYPE }}_${{ MODE }}
+              displayName: Remove trial ${{ OS_NAME }}/${{ BUILD_SYSTEM }}/${{ BUILD_TYPE }}/${{ MODE }} WATS outputs 
+
+  - script: |
+      rm -rf wats_outputs/wats_Ubuntu_Make_Debug_serial
+    displayName: Remove reference Ubuntu/Make/Debug/serial WATS outputs 
+
+  - ${{ each DOMAIN in parameters.DOMAINS }}:   
+    - ${{ each TIME in parameters.TIMES }}:
+      - script: |
+          python wats/wats/plots.py plot --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --plots-dir wats_plots/single_ref/${{ DOMAIN }}/T${{ TIME }}
+        displayName: 'Run WATS: create Ubuntu/Make/Debug/serial reference plots for domain ${{ DOMAIN }} - T${{ TIME }}'
+
+  - script: |
+      rm -rf wats_stats/
+    displayName: Remove WATS stats files
+  
+  # Plot: CMake vs Make
+
+  - ${{ each OS_NAME in parameters.OS_NAMES }}:
+    - ${{ each BUILD_TYPE in parameters.BUILD_TYPES }}:
+      - ${{ each MODE in parameters.MODES }}:
+        # For Windows, since there is no Make-based build, skip that combination.
+        - ${{ if not(eq(OS_NAME, 'Windows')) }}:
+
+          - task: DownloadBuildArtifacts@0
+            displayName: Retrieve ${{ OS_NAME }}/{Make,CMake}/${{ BUILD_TYPE }}/${{ MODE }} WATS outputs
+            inputs:
+              downloadPath: wats_outputs
+              downloadType: specific
+              itemPattern: 'wats_${{ OS_NAME }}_*_${{ BUILD_TYPE }}_${{ MODE }}/**'
+
+          - ${{ each DOMAIN in parameters.DOMAINS }}:
+            - ${{ each TIME in parameters.TIMES }}:
+              - script: |
+                  set -ex
+                  O=wats_outputs
+                  W="python wats/wats/plots.py compute --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --filter ${{ DOMAIN }} --time-idx ${{ TIME }} --append"
+                  $W $O/wats_${{ OS_NAME }}_Make_${{ BUILD_TYPE }}_${{ MODE }} $O/wats_${{ OS_NAME }}_CMake_${{ BUILD_TYPE }}_${{ MODE }}
+                displayName: 'Run WATS: compute plot stats ${{ OS_NAME }}/Make/${{ BUILD_TYPE }}/${{ MODE }} vs ${{ OS_NAME }}/CMake/${{ BUILD_TYPE }}/${{ MODE }} for domain ${{ DOMAIN }} - T${{ TIME }}'
+
+          - script: |
+              rm -rf wats_outputs/wats_${{ OS_NAME }}_*_${{ BUILD_TYPE }}_${{ MODE }}
+            displayName: Remove ${{ OS_NAME }}/{Make,CMake}/${{ BUILD_TYPE }}/${{ MODE }} WATS outputs 
+
+  - ${{ each DOMAIN in parameters.DOMAINS }}:   
+    - ${{ each TIME in parameters.TIMES }}:
+      - script: |
+          python wats/wats/plots.py plot --stats-dir wats_stats/${{ DOMAIN }}/T${{ TIME }} --plots-dir wats_plots/make_cmake/${{ DOMAIN }}/T${{ TIME }}
+        displayName: 'Run WATS: create Make vs CMake plots for domain ${{ DOMAIN }} - T${{ TIME }}'
+
+  - task: PublishBuildArtifacts@1
+    displayName: Store WATS plots
+    inputs:
+      pathtoPublish: wats_plots
+      artifactName: wats_plots