Cirrus: Migrate off papr + implement VM testing

Signed-off-by: Chris Evich <cevich@redhat.com>
containers · Feb 6, 2020 · 9086862 · 9086862
1 parent 8869618
commit 9086862
Show file tree

Hide file tree

Showing 9 changed files with 346 additions and 210 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -5,24 +5,31 @@ env:
     ####
     #### Global variables used for all tasks
     ####
+    GOPATH: "/var/tmp/go"
+    GOSRC: "${GOPATH}/src/github.com/containers/buildah"
     # Overrides default location (/tmp/cirrus) for repo clone
-    CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/buildah"
+    CIRRUS_WORKING_DIR: "${GOSRC}"
     # Shell used to execute all script commands
     CIRRUS_SHELL: "/bin/bash"
     # Automation script path relative to $CIRRUS_WORKING_DIR)
     SCRIPT_BASE: "./contrib/cirrus"
     # No need to go crazy, but grab enough to cover most PRs
     CIRRUS_CLONE_DEPTH: 50
+    # Unless set by in_podman.sh, default to operating outside of a podman container
+    IN_PODMAN: 'false'
+    # Not cross-compiling by default
+    CROSS_TARGET: ""
 
     ####
     #### Cache-image names to test with
     ####
     # GCE project where images live
     IMAGE_PROJECT: "libpod-218412"
+    # TODO: Setting up from base-images is very inefficient, use libpod's cache-images instead?
     FEDORA_CACHE_IMAGE_NAME: "fedora-cloud-base-30-1-2-1565360543"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-cloud-base-29-1-2-1565360543"
-    UBUNTU_CACHE_IMAGE_NAME: "ubuntu-1904-disco-v20190724"          # Latest
-    PRIOR_UBUNTU_CACHE_IMAGE_NAME: "ubuntu-1804-bionic-v20190722a"  # LTS
+    UBUNTU_CACHE_IMAGE_NAME: "ubuntu-1904-disco-v20190724"
+    PRIOR_UBUNTU_CACHE_IMAGE_NAME: "ubuntu-1804-bionic-v20190722a"
 
     ####
     #### Command variables to help avoid duplication
@@ -50,29 +57,43 @@ gce_instance:
     image_name: "${FEDORA_CACHE_IMAGE_NAME}"
 
 
-testing_task:
-    gce_instance:  # Only need to specify differences from defaults (above)
-        matrix:  # Duplicate this task for each matrix product.
-            image_name: "${FEDORA_CACHE_IMAGE_NAME}"
-            image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
-            image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
-            image_name: "${PRIOR_UBUNTU_CACHE_IMAGE_NAME}"
+# Update metadata on VM images referenced by this repository state
+meta_task:
 
-    # Separate scripts for separate outputs, makes debugging easier.
-    setup_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/setup.sh |& ${_TIMESTAMP}'
-    build_and_test_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/build_and_test.sh |& ${_TIMESTAMP}'
+    container:
+        image: "quay.io/libpod/imgts:latest"  # see contrib/imgts
+        cpu: 1
+        memory: 1
+
+    env:
+        # Space-separated list of images used by this repository state
+        IMGNAMES: |-
+            ${FEDORA_CACHE_IMAGE_NAME}
+            ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
+            ${UBUNTU_CACHE_IMAGE_NAME}
+            ${PRIOR_UBUNTU_CACHE_IMAGE_NAME}
+        BUILDID: "${CIRRUS_BUILD_ID}"
+        REPOREF: "${CIRRUS_CHANGE_IN_REPO}"
+        GCPJSON: ENCRYPTED[d3614d6f5cc0e66be89d4252b3365fd84f14eee0259d4eb47e25fc0bc2842c7937f5ee8c882b7e547b4c5ec4b6733b14]
+        GCPNAME: ENCRYPTED[8509e6a681b859479ce6aa275bd3c4ac82de5beec6df6057925afc4cd85b7ef2e879066ae8baaa2d453b82958e434578]
+        GCPPROJECT: ENCRYPTED[cc09b62d0ec6746a3df685e663ad25d9d5af95ef5fd843c96f3d0ec9d7f065dc63216b9c685c9f43a776a1d403991494]
+        CIRRUS_CLONE_DEPTH: 1  # source not used
+
+    script: '/usr/local/bin/entrypoint.sh |& ${_TIMESTAMP}'
 
-    # Log collection when job was successful
-    df_script: '${_DFCMD} || true'
-    rh_audit_log_script: '${_RAUDITCMD} || true'
-    ubuntu_audit_log_script: '${_UAUDITCMD} || true'
-    journal_log_script: '${_JOURNALCMD} || true'
 
-    on_failure:  # Script names must be different from above
-        failure_df_script: '${_DFCMD} || true'
-        failure_rh_audit_log_script: '${_RAUDITCMD} || true'
-        failure_ubuntu_audit_log_script: '${_UAUDITCMD} || true'
-        failure_journal_log_script: '${_JOURNALCMD} || true'
+gate_task:
+
+    timeout_in: 20m
+
+    setup_script: '${SCRIPT_BASE}/setup.sh |& ${_TIMESTAMP}'
+    build_script: '${SCRIPT_BASE}/build.sh |& ${_TIMESTAMP}'
+    validate_test_script: '${SCRIPT_BASE}/test.sh validate |& ${_TIMESTAMP}'
+    # FIXME: some tests broken
+    # unit_test_script: '${SCRIPT_BASE}/test.sh unit |& ${_TIMESTAMP}'
+
+    binary_artifacts:
+        path: ./bin/*
 
 
 # This task runs `make vendor` followed by ./hack/tree_status.sh to check
@@ -81,48 +102,92 @@ testing_task:
 # in sync at all times.
 vendor_task:
 
-    only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
-
     env:
         CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/buildah"
-        GOPATH: "/go"
-        GOSRC: "/go/src/github.com/containers/buildah"
+        GOPATH: "/var/tmp/go"
+        GOSRC: "/var/tmp/go/src/github.com/containers/buildah"
 
     # Runs within Cirrus's "community cluster"
     container:
         image: docker.io/library/golang:1.13
         cpu: 1
         memory: 1
 
-    timeout_in: 30m
+    timeout_in: 5m
 
     vendor_script:
-        - 'cd ${CIRRUS_WORKING_DIR} && make vendor'
-        - 'cd ${CIRRUS_WORKING_DIR} && ./hack/tree_status.sh'
+        - 'make vendor'
+        - './hack/tree_status.sh'
 
-# Update metadata on VM images referenced by this repository state
-meta_task:
+
+cross_task:
 
     depends_on:
-        - "vendor"
+        - gate
+        - vendor
 
     container:
-        image: "quay.io/libpod/imgts:latest"  # see contrib/imgts
-        cpu: 1
-        memory: 1
+        image: registry.fedoraproject.org/fedora:30
 
     env:
-        # Space-separated list of images used by this repository state
-        IMGNAMES: |-
-            ${FEDORA_CACHE_IMAGE_NAME}
-            ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
-            ${UBUNTU_CACHE_IMAGE_NAME}
-            ${PRIOR_UBUNTU_CACHE_IMAGE_NAME}
-        BUILDID: "${CIRRUS_BUILD_ID}"
-        REPOREF: "${CIRRUS_CHANGE_IN_REPO}"
-        GCPJSON: ENCRYPTED[d3614d6f5cc0e66be89d4252b3365fd84f14eee0259d4eb47e25fc0bc2842c7937f5ee8c882b7e547b4c5ec4b6733b14]
-        GCPNAME: ENCRYPTED[8509e6a681b859479ce6aa275bd3c4ac82de5beec6df6057925afc4cd85b7ef2e879066ae8baaa2d453b82958e434578]
-        GCPPROJECT: ENCRYPTED[cc09b62d0ec6746a3df685e663ad25d9d5af95ef5fd843c96f3d0ec9d7f065dc63216b9c685c9f43a776a1d403991494]
-        CIRRUS_CLONE_DEPTH: 1  # source not used
+        matrix:
+            CROSS_TARGET: darwin
 
-    script: '/usr/local/bin/entrypoint.sh |& ${_TIMESTAMP}'
+    setup_script: '${SCRIPT_BASE}/setup.sh |& ${_TIMESTAMP}'
+    build_script: '${SCRIPT_BASE}/build.sh |& ${_TIMESTAMP}'
+
+    binary_artifacts:
+        path: ./bin/*
+
+
+testing_task:
+
+    depends_on:
+        - gate
+        - vendor
+
+    gce_instance:  # Only need to specify differences from defaults (above)
+        matrix:  # Duplicate this task for each matrix product.
+            image_name: "${FEDORA_CACHE_IMAGE_NAME}"
+            image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
+            image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
+            image_name: "${PRIOR_UBUNTU_CACHE_IMAGE_NAME}"
+
+    # Separate scripts for separate outputs, makes debugging easier.
+    setup_script: '${SCRIPT_BASE}/setup.sh |& ${_TIMESTAMP}'
+    build_script: '${SCRIPT_BASE}/build.sh |& ${_TIMESTAMP}'
+    integration_test_script: '${SCRIPT_BASE}/test.sh integration |& ${_TIMESTAMP}'
+
+    binary_artifacts:
+        path: ./bin/*
+
+    always: &standardlogs
+        df_script: '${_DFCMD} || true'
+        rh_audit_log_script: '${_RAUDITCMD} || true'
+        ubuntu_audit_log_script: '${_UAUDITCMD} || true'
+        journal_log_script: '${_JOURNALCMD} || true'
+
+
+testing_in_podman_task:
+
+    depends_on:
+        - gate
+        - vendor
+
+    env:
+        # This is key, it causes the scripts to re-execute themselves inside a container.
+        IN_PODMAN: 'true'
+        BUILDAH_ISOLATION: 'chroot'
+        STORAGE_DRIVER: 'vfs'
+        STORAGE_OPTIONS: ''
+
+    # Separate scripts for separate outputs, makes debugging easier.
+    setup_script: '${SCRIPT_BASE}/setup.sh |& ${_TIMESTAMP}'
+    build_script: '${SCRIPT_BASE}/build.sh |& ${_TIMESTAMP}'
+    integration_test_script: '${SCRIPT_BASE}/test.sh integration |& ${_TIMESTAMP}'
+
+    binary_artifacts:
+        path: ./bin/*
+
+    always:
+        <<: *standardlogs
diff --git a/.papr.sh b/.papr.sh
diff --git a/.papr.yml b/.papr.yml
diff --git a/contrib/cirrus/build.sh b/contrib/cirrus/build.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+
+source $(dirname $0)/lib.sh
+
+req_env_var IN_PODMAN IN_PODMAN_NAME GOSRC
+
+if [[ "$IN_PODMAN" == "true" ]]
+then
+    cd $GOSRC
+    in_podman --rm $IN_PODMAN_NAME $0
+else
+    cd $GOSRC
+    echo "Compiling buildah"
+    showrun make $CROSS_TARGET ${CROSS_TARGET:+CGO_ENABLED=0}
+    mkdir -p bin
+
+    echo "Installing buildah"
+    if [[ -z "$CROSS_TARGET" ]]
+    then
+        ln -v buildah bin/buildah
+        showrun make install PREFIX=/usr
+    else
+        ln -v buildah.${CROSS_TARGET} bin/buildah
+    fi
+fi
diff --git a/contrib/cirrus/build_and_test.sh b/contrib/cirrus/build_and_test.sh