Merge branch 'master' into performance-baselines

.github/workflows/container.yml

@@ -0,0 +1,58 @@
+name: container build/publish
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'docker/**'
+
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'docker/**'
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Only build container if there has been a change.
+  build-containers:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ghcr.io/ESMCI/cime
+          flavor: |
+            latest=auto
+          tags: |
+            type=sha
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          target: base
+          context: docker/
+          push: ${{ github.event == 'push' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/testing.yml

@@ -4,14 +4,20 @@ on:
   push:
     branches:
       - master
-    paths-ignore:
-      - 'doc/**'
+    paths:
+      - 'CIME/**'
+      - 'scripts/**'
+      - 'tools/**'
+      - 'utils/**'
 
   pull_request:
     branches:
       - master
-    paths-ignore:
-      - 'doc/**'
+    paths:
+      - 'CIME/**'
+      - 'scripts/**'
+      - 'tools/**'
+      - 'utils/**'
 
 concurrency:
   group: ${{ github.ref }}
@@ -43,64 +49,16 @@ jobs:
 
           pre-commit run -a
 
-  # Check if there has been a change to any file under docker/
-  get-docker-changes:
-    runs-on: ubuntu-latest
-    outputs:
-      any_changed: ${{ steps.get-changed-files.outputs.any_changed }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-      - name: Get changed files
-        id: get-changed-files
-        uses: tj-actions/changed-files@v29
-        with:
-          files: docker
-
-  # Only build container if there has been a change.
-  build-containers:
-    runs-on: ubuntu-latest
-    needs: get-docker-changes
-    if: ${{ needs.get-docker-changes.outputs.any_changed == 'true' }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: jasonb87/cime
-          tags: |
-            type=raw,value=latest
-            type=sha,prefix={{ date 'YYYYMMDD' }}_,format=short
-      - name: Build and push
-        uses: docker/build-push-action@v3
-        with:
-          target: base
-          context: docker/
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=registry,ref=jasonb87/cime:buildcache
-          cache-to: type=registry,ref=jasonb87/cime:buildcache,mode=max
-
   # Runs unit testing under different python versions.
   unit-testing:
     runs-on: ubuntu-latest
-    needs: build-containers
     if: ${{ always() && ! cancelled() }}
-    container: jasonb87/cime:latest
+    container:
+      image: ghcr.io/esmci/cime:latest
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      options: '--pull=always'
     strategy:
       matrix:
         python-version: ['3.8', '3.9', '3.10']
@@ -134,9 +92,13 @@ jobs:
   # Run system tests
   system-testing:
     runs-on: ubuntu-latest
-    needs: build-containers
     if: ${{ always() && ! cancelled() }}
-    container: jasonb87/cime:latest
+    container:
+      image: ghcr.io/esmci/cime:latest
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      options: '--pull=always'
     strategy:
       matrix:
         model: ["e3sm", "cesm"]

CIME/XML/env_batch.py

@@ -795,20 +795,10 @@ def submit_jobs(
         batch_job_id = None
         for _ in range(num_submit):
             for job, dependency in jobs:
-                if dependency is not None:
-                    deps = dependency.split()
-                else:
-                    deps = []
-                dep_jobs = []
-                if user_prereq is not None:
-                    dep_jobs.append(user_prereq)
-                for dep in deps:
-                    if dep in depid.keys() and depid[dep] is not None:
-                        dep_jobs.append(str(depid[dep]))
-                if prev_job is not None:
-                    dep_jobs.append(prev_job)
+                dep_jobs = get_job_deps(dependency, depid, prev_job, user_prereq)
 
                 logger.debug("job {} depends on {}".format(job, dep_jobs))
+
                 result = self._submit_single_job(
                     case,
                     job,
@@ -1399,3 +1389,41 @@ def make_all_batch_files(self, case):
                         input_batch_script, job
                     )
                 )
+
+
+def get_job_deps(dependency, depid, prev_job=None, user_prereq=None):
+    """
+    Gather list of job batch ids that a job depends on.
+
+    Parameters
+    ----------
+    dependency : str
+        List of dependent job names.
+    depid : dict
+        Lookup where keys are job names and values are the batch id.
+    user_prereq : str
+        User requested dependency.
+
+    Returns
+    -------
+    list
+        List of batch ids that job depends on.
+    """
+    deps = []
+    dep_jobs = []
+
+    if user_prereq is not None:
+        dep_jobs.append(user_prereq)
+
+    if dependency is not None:
+        # Match all words, excluding "and" and "or"
+        deps = re.findall(r"\b(?!and\b|or\b)\w+(?:\.\w+)?\b", dependency)
+
+        for dep in deps:
+            if dep in depid and depid[dep] is not None:
+                dep_jobs.append(str(depid[dep]))
+
+    if prev_job is not None:
+        dep_jobs.append(prev_job)
+
+    return dep_jobs

CIME/case/case_st_archive.py

@@ -1186,7 +1186,9 @@ def test_env_archive(self, testdir="env_archive_test"):
 
     for comp_archive_spec in comp_archive_specs:
         comp_expected = archive.get(comp_archive_spec, "compname")
-        if comp_expected == "ww3":
+        # Rename ww3 component when case and archive names don't match,
+        # specific to CESM.
+        if comp_expected == "ww3" and "ww" in comps_in_case:
             comp_expected = "ww"
         comp_class = archive.get(comp_archive_spec, "compclass").upper()
         if comp_class in components: