Merge pull request #2214 from cevich/cirrus_imgts_activate

Cirrus: Use imgts container to record metadata (needs #2213)

.cirrus.yml

@@ -27,11 +27,17 @@ env:
     ####
     #### Cache-image names to test with
     ###
+    ACTIVE_CACHE_IMAGE_NAMES: >-
+        fedora-29-libpod-d6d53e40
+        fedora-28-libpod-d6d53e40
+        ubuntu-18-libpod-d6d53e40
+        rhel-7-libpod-7f4cd1f7
+        image-builder-image-1541772081
     FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-d6d53e40"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-d6d53e40"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-d6d53e40"
+    PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-7f4cd1f7"
     # RHEL_CACHE_IMAGE_NAME: "rhel-8-notready"
-    PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-d6d53e40"
     # CENTOS_CACHE_IMAGE_NAME: "centos-7-notready"
 
     ####
@@ -108,6 +114,7 @@ gating_task:
     gate_script:
         - '/usr/local/bin/entrypoint.sh validate'
         - '/usr/local/bin/entrypoint.sh lint'
+        - '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/test/test_dot_cirrus_yaml.py'
 
     # This task runs `make vendor` followed by ./hack/tree_status.sh to check
     # whether the git tree is clean.  The reasoning for that is to make sure
@@ -149,6 +156,30 @@ build_each_commit_task:
         - env GOPATH=/var/tmp/go/ make build-all-new-commits GIT_BASE_BRANCH=origin/$CIRRUS_BASE_BRANCH
 
 
+# Update metadata on VM images referenced by this repository state
+meta_task:
+
+    depends_on:
+        - "gating"
+
+    container:
+        image: "quay.io/libpod/imgts:latest"  # see contrib/imgts
+        cpu: 1
+        memory: 1
+
+    env:
+        # Space-separated list of images used by this repository state
+        IMGNAMES: "${ACTIVE_CACHE_IMAGE_NAMES}"
+        BUILDID: "${CIRRUS_BUILD_ID}"
+        REPOREF: "${CIRRUS_CHANGE_IN_REPO}"
+        GCPJSON: ENCRYPTED[950d9c64ad78f7b1f0c7e499b42dc058d2b23aa67e38b315e68f557f2aba0bf83068d4734f7b1e1bdd22deabe99629df]
+        GCPNAME: ENCRYPTED[b05d469a0dba8cb479cb00cc7c1f6747c91d17622fba260a986b976aa6c817d4077eacffd4613d6d5f23afc4084fab1d]
+        GCPPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
+        CIRRUS_CLONE_DEPTH: 1  # source not used
+
+    script: /usr/local/bin/entrypoint.sh
+
+
 # This task does the unit and integration testing for every platform
 testing_task:
 

contrib/cirrus/README.md

@@ -33,6 +33,17 @@ task (pass or fail) is set based on the exit status of the last script to execut
    the vendor.conf, the code and the vendored packages in ./vendor are in sync
    at all times.
 
+### ``meta`` Task
+
+***N/B: Steps below are performed by automation***
+
+1. Launch a container built from definition in ``./contrib/imgts``.
+
+2. Update VM Image metadata to help track usage across all automation.
+
+4. Always exits successfully unless there's a major problem.
+
+
 ### ``testing`` Task
 
 ***N/B: Steps below are performed by automation***

contrib/cirrus/test/test_dot_cirrus_yaml.py

@@ -0,0 +1,78 @@
+#!/bin/env python3
+
+import sys
+import os
+import os.path
+import unittest
+import warnings
+import yaml
+
+class TestCaseBase(unittest.TestCase):
+
+    SCRIPT_PATH = os.path.realpath((os.path.dirname(sys.argv[0])))
+    CIRRUS_WORKING_DIR = os.environ.get('CIRRUS_WORKING_DIR',
+                                        '{0}/../../../'.format(SCRIPT_PATH))
+
+    def setUp(self):
+        os.chdir(self.CIRRUS_WORKING_DIR)
+
+
+class TestCirrusYAML(TestCaseBase):
+
+    IMAGE_NAME_SUFFIX = '_CACHE_IMAGE_NAME'
+    ACTIVE_IMAGES_NAME = 'ACTIVE_CACHE_IMAGE_NAMES'
+
+    def setUp(self):
+        TestCirrusYAML._cirrus = None
+        super().setUp()
+
+    @property
+    def cirrus(self):
+        if TestCirrusYAML._cirrus is None:
+            with warnings.catch_warnings():
+                warnings.filterwarnings("ignore",category=DeprecationWarning)
+                with open('.cirrus.yml', "r") as dot_cirrus_dot_yaml:
+                    TestCirrusYAML._cirrus = yaml.load(dot_cirrus_dot_yaml)
+        return TestCirrusYAML._cirrus
+
+    def _assert_get_cache_image_names(self, env):
+        inames = set([key for key in env.keys()
+                      if key.endswith(self.IMAGE_NAME_SUFFIX)])
+        self.assertNotEqual(inames, set())
+
+        ivalues = set([value for key, value in env.items()
+                       if key in inames])
+        self.assertNotEqual(ivalues, set())
+        return ivalues
+
+    def _assert_get_subdct(self, key, dct):
+        self.assertIn(key, dct)
+        return dct[key]
+
+    def test_parse_yaml(self):
+        self.assertIsInstance(self.cirrus, dict)
+
+    def test_active_cache_image_names(self):
+        env = self._assert_get_subdct('env', self.cirrus)
+        acin = self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env)
+
+        for ivalue in self._assert_get_cache_image_names(env):
+            self.assertIn(ivalue, acin,
+                          "The '{}' sub-key of 'env' should contain this among"
+                          " its space-separated values."
+                          "".format(self.ACTIVE_IMAGES_NAME))
+
+
+    def test_cache_image_names_active(self):
+        env = self._assert_get_subdct('env', self.cirrus)
+        ivalues = self._assert_get_cache_image_names(env)
+
+        for avalue in set(self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env).split()):
+            self.assertIn(avalue, ivalues,
+                          "All space-separated values in the '{}' sub-key"
+                          " of 'env' must also be used in a key with a '{}' suffix."
+                          "".format(self.ACTIVE_IMAGES_NAME, self.IMAGE_NAME_SUFFIX))
+
+
+if __name__ == '__main__':
+    unittest.main(failfast=True, catchbreak=True, verbosity=0)

contrib/gate/Dockerfile

@@ -29,6 +29,7 @@ RUN dnf -y install \
       python3-dateutil \
       python3-psutil \
       python3-pytoml \
+      python3-pyyaml \
       python3-varlink \
       skopeo-containers \
       slirp4netns \