DAOS-623 test: Don't add repos that are already present (#5035)

This bogs down dnf and saves significant provisioning time.
daos-stack · Mar 19, 2021 · 88872a5 · 88872a5
1 parent 6be12c4
commit 88872a5
Show file tree

Hide file tree

Showing 5 changed files with 95 additions and 94 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -67,8 +67,7 @@ def functional_post_always() {
 }
 
 String get_daos_packages() {
-    Map stage_info = parseStageInfo()
-    return get_daos_packages(stage_info['target'])
+    return get_daos_packages(parseStageInfo()['target'])
 }
 
 String get_daos_packages(String distro) {
@@ -85,15 +84,14 @@ String get_daos_packages(String distro) {
 }
 
 String pr_repos() {
-    Map stage_info = parseStageInfo()
-    return pr_repos(stage_info['target'])
+    return pr_repos(parseStageInfo()['target'])
 }
 
 String pr_repos(String distro) {
     String repos = ""
-    if (distro == 'centos7') {
+    if (distro.startsWith('el7') || distro.startsWith('centos7')) {
         repos = cachedCommitPragma(pragma: 'PR-repos-el7')
-    } else if (distro == 'leap15') {
+    } else if (distro.startsWith('leap15')) {
         repos = cachedCommitPragma(pragma: 'PR-repos-leap15')
     } else if (distro.startsWith('ubuntu20')) {
         repos = cachedCommitPragma(pragma: 'PR-repos-ubuntu20', cache: commit_pragma_cache)
@@ -123,8 +121,7 @@ String hw_distro_target() {
             return hw_distro('large')
         }
     }
-    Map stage_info = parseStageInfo()
-    return stage_info['target']
+    return parseStageInfo()['target']
 }
 
 String daos_repos() {
@@ -142,7 +139,8 @@ String unit_packages() {
     if (env.STAGE_NAME.contains('Bullseye')) {
         need_qb = true
     }
-    if (stage_info['target'] == 'centos7') {
+    if (stage_info['target'].startsWith('el7') ||
+        stage_info['target'].startsWith('centos7')) {
         String packages =  'gotestsum openmpi3 ' +
                            'hwloc-devel argobots ' +
                            'fuse3-libs fuse3 ' +
@@ -183,8 +181,7 @@ def cachedCommitPragma(Map config) {
 }
 
 String daos_packages_version() {
-    stage_info = parseStageInfo()
-    return daos_packages_version(stage_info['target'])
+    return daos_packages_version(parseStageInfo()['target'])
 }
 
 String daos_packages_version(String distro) {
@@ -195,9 +192,9 @@ String daos_packages_version(String distro) {
         String dist = ""
         if (version.indexOf('-') > -1) {
             // only tack on the %{dist} if the release was specified
-            if (distro == "centos7") {
+            if (distro.startsWith('el7') || distro.startsWith('centos7')) {
                 dist = ".el7"
-            } else if (distro == "leap15") {
+            } else if (distro.startsWith('leap15')) {
                 dist = ".suse.lp152"
             }
         }
@@ -254,15 +251,15 @@ String functional_packages(String distro) {
                   "MACSio-mpich " +
                   "MACSio-openmpi3 " +
                   "mpifileutils-mpich-daos-1 "
-    if (distro == "leap15") {
+    if (distro.startsWith('leap15')) {
         return daos_pkgs + pkgs
-    } else if (distro == "centos7") {
+    } else if (distro.startsWith('el7') || distro.startsWith('centos7')) {
         // need to exclude openmpi until we remove it from the repo
         return  "--exclude openmpi " + daos_pkgs + pkgs
     } else if (distro.startsWith('ubuntu20')) {
         return daos_pkgs + " openmpi-bin ndctl fio"
     } else {
-        error 'functional_packages not implemented for ' + stage_info['target']
+        error 'functional_packages not implemented for ' + distro
     }
 }
 
@@ -478,11 +475,11 @@ String quick_build_deps(String distro, always=false) {
             return ""
         }
     }
-    if (distro == "leap15") {
+    if (distro.startsWith('leap15')) {
         rpmspec_args = "--define dist\\ .suse.lp152 " +
                        "--undefine rhel " +
                        "--define suse_version\\ 1502"
-    } else if (distro == "centos7") {
+    } else if (distro.startsWith('el7') || distro.startsWith('centos7')) {
         rpmspec_args = "--undefine suse_version " +
                        "--define rhel\\ 7"
     } else {
@@ -1313,7 +1310,7 @@ pipeline {
                         expression { ! skip_ftest('el7') }
                     }
                     agent {
-                        label 'ci_vm9'
+                        label 'stage_vm9'
                     }
                     steps {
                         functionalTest inst_repos: daos_repos(),
@@ -1332,7 +1329,7 @@ pipeline {
                         expression { ! skip_ftest('leap15') }
                     }
                     agent {
-                        label 'ci_vm9'
+                        label 'stage_vm9'
                     }
                     steps {
                         functionalTest inst_repos: daos_repos(),

diff --git a/ci/provisioning/post_provision_config.sh b/ci/provisioning/post_provision_config.sh
@@ -17,19 +17,19 @@ DSG_REPO_var="DAOS_STACK_${DISTRO}_GROUP_REPO"
 
 clush -B -l root -w "$NODESTRING" -c ci_key* --dest=/tmp/
 
-clush -B -S -l root -w "$NODESTRING" \
-    "MY_UID=$(id -u)
-    CONFIG_POWER_ONLY=$CONFIG_POWER_ONLY
-    INST_REPOS=\"$INST_REPOS\"
-    INST_RPMS=\$(eval echo $INST_RPMS)
-    GPG_KEY_URLS=\"$GPG_KEY_URLS\"
-    REPOSITORY_URL=\"$REPOSITORY_URL\"
-    JENKINS_URL=\"$JENKINS_URL\"
-    DAOS_STACK_LOCAL_REPO=\"${!DSL_REPO_var}\"
-    DAOS_STACK_GROUP_REPO=\"${!DSG_REPO_var:-}\"
-    DISTRO=\"$DISTRO\"
-    $(cat ci/provisioning/post_provision_config_nodes_"${DISTRO}".sh)
-    $(cat ci/provisioning/post_provision_config_nodes.sh)"
+time clush -B -S -l root -w "$NODESTRING" \
+           "MY_UID=$(id -u)
+           CONFIG_POWER_ONLY=$CONFIG_POWER_ONLY
+           INST_REPOS=\"$INST_REPOS\"
+           INST_RPMS=\$(eval echo $INST_RPMS)
+           GPG_KEY_URLS=\"$GPG_KEY_URLS\"
+           REPOSITORY_URL=\"$REPOSITORY_URL\"
+           JENKINS_URL=\"$JENKINS_URL\"
+           DAOS_STACK_LOCAL_REPO=\"${!DSL_REPO_var}\"
+           DAOS_STACK_GROUP_REPO=\"${!DSG_REPO_var:-}\"
+           DISTRO=\"$DISTRO\"
+           $(cat ci/provisioning/post_provision_config_nodes_"${DISTRO}".sh)
+           $(cat ci/provisioning/post_provision_config_nodes.sh)"
 
 git log --format=%s -n 1 HEAD | ssh -i ci_key -l jenkins "${NODELIST%%,*}" \
                                     "cat >/tmp/commit_title"

diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh
@@ -14,20 +14,44 @@ url_to_repo() {
 }
 
 add_repo() {
-    local repo="$1"
-    local gpg_check="${2:-true}"
+    local match="$1"
+    local add_repo="$2"
+    local gpg_check="${3:-true}"
 
-    if [ -n "$repo" ]; then
-        repo="${REPOSITORY_URL}${repo}"
-        if ! dnf repolist | grep "$(url_to_repo "$repo")"; then
-            dnf config-manager --add-repo="${repo}"
+    if [ -z "$match" ]; then
+        # we cannot try to add a repo that has no match
+        return
+    fi
+
+    local repo
+    # see if a package we know is in the repo is present
+    if repo=$(dnf repoquery --qf "%{repoid}" "$1" 2>/dev/null | grep ..\*); then
+        DNF_REPO_ARGS+=" --enablerepo=$repo"
+    else
+        local repo_url="${REPOSITORY_URL}${add_repo}"
+        local repo_name
+        repo_name=$(url_to_repo "$repo_url")
+        if ! dnf repolist | grep "$repo_name"; then
+            dnf config-manager --add-repo="${repo_url}" >&2
             if ! $gpg_check; then
-                disable_gpg_check "$repo"
+                disable_gpg_check "$add_repo" >&2
             fi
         fi
+        DNF_REPO_ARGS+=" --enablerepo=$repo_name"
     fi
 }
 
+add_group_repo() {
+    local match="$1"
+
+    add_repo "$match" "$DAOS_STACK_GROUP_REPO"
+    group_repo_post
+}
+
+add_local_repo() {
+    add_repo 'argobots' "$DAOS_STACK_LOCAL_REPO" false
+}
+
 disable_gpg_check() {
     local url="$1"
 

diff --git a/ci/provisioning/post_provision_config_nodes_EL_7.sh b/ci/provisioning/post_provision_config_nodes_EL_7.sh
@@ -3,7 +3,7 @@
 REPOS_DIR=/etc/yum.repos.d
 DISTRO_NAME=centos7
 LSB_RELEASE=redhat-lsb-core
-PYTHON_MACROS_RPM=("python3-rpm-macros")
+EXCLUDE_UPGRADE=fuse,mercury,daos,daos-\*
 
 timeout_yum() {
     local timeout="$1"
@@ -33,13 +33,13 @@ bootstrap_dnf() {
 }
 
 group_repo_post() {
-    # nothing for EL7
+    # Nothing to do for EL
     :
 }
 
 distro_custom() {
     # shellcheck disable=SC2086
-    dnf -y $dnf_repo_args install python3
+    time dnf -y install python3
 
     if [ ! -e /usr/bin/pip3 ] &&
        [ -e /usr/bin/pip3.6 ]; then
@@ -49,6 +49,7 @@ distro_custom() {
        [ -e /usr/bin/python3.6 ]; then
         ln -s python3.6 /usr/bin/python3
     fi
+
     # install the debuginfo repo in case we get segfaults
     cat <<"EOF" > $REPOS_DIR/CentOS-Debuginfo.repo
 [core-0-debuginfo]
@@ -60,9 +61,9 @@ enabled=0
 EOF
 
     # force install of avocado 69.x
-    dnf -y erase avocado{,-common}                                        \
-           python2-avocado{,-plugins-{output-html,varianter-yaml-to-mux}} \
-           python36-PyYAML
+    dnf -y erase avocado{,-common}                                              \
+                 python2-avocado{,-plugins-{output-html,varianter-yaml-to-mux}} \
+                 python36-PyYAML
     pip3 install --upgrade pip
     pip3 install "avocado-framework<70.0"
     pip3 install "avocado-framework-plugin-result-html<70.0"
@@ -78,23 +79,19 @@ post_provision_config_nodes() {
 
     if $CONFIG_POWER_ONLY; then
         rm -f $REPOS_DIR/*.hpdd.intel.com_job_daos-stack_job_*_job_*.repo
-        dnf -y erase fio fuse ior-hpc mpich-autoload               \
+        time dnf -y erase fio fuse ior-hpc mpich-autoload               \
                      ompi argobots cart daos daos-client dpdk      \
                      fuse-libs libisa-l libpmemobj mercury mpich   \
                      openpa pmix protobuf-c spdk libfabric libpmem \
                      libpmemblk munge-libs munge slurm             \
                      slurm-example-configs slurmctld slurm-slurmmd
     fi
 
-    local dnf_repo_args="--disablerepo=*"
-
-    add_repo "$DAOS_STACK_GROUP_REPO"
-    group_repo_post
-
-    add_repo "${DAOS_STACK_LOCAL_REPO}" false
-
-    # TODO: this should be per repo for the above two repos
-    dnf_repo_args+=" --enablerepo=repo.dc.hpdd.intel.com_repository_*"
+    time dnf repolist
+    # the group repo is always on the test image
+    #add_group_repo
+    add_local_repo
+    time dnf repolist
 
     if [ -n "$INST_REPOS" ]; then
         local repo
@@ -112,23 +109,19 @@ post_provision_config_nodes() {
             local repo_url="${JENKINS_URL}"job/daos-stack/job/"${repo}"/job/"${branch//\//%252F}"/"${build_number}"/artifact/artifacts/$DISTRO_NAME/
             dnf config-manager --add-repo="${repo_url}"
             disable_gpg_check "$repo_url"
-            # TODO: this should be per repo in the above loop
-            if [ -n "$INST_REPOS" ]; then
-                dnf_repo_args+=",build.hpdd.intel.com_job_daos-stack*"
-            fi
         done
     fi
     if [ -n "$INST_RPMS" ]; then
         # shellcheck disable=SC2086
-        dnf -y erase $INST_RPMS
+        time dnf -y erase $INST_RPMS
     fi
     rm -f /etc/profile.d/openmpi.sh
     rm -f /tmp/daos_control.log
-    dnf -y install $LSB_RELEASE
+    time dnf -y install $LSB_RELEASE
 
     # shellcheck disable=SC2086
     if [ -n "$INST_RPMS" ] &&
-       ! dnf -y $dnf_repo_args install $INST_RPMS; then
+       ! time dnf -y install $INST_RPMS; then
         rc=${PIPESTATUS[0]}
         dump_repos
         exit "$rc"
@@ -138,17 +131,14 @@ post_provision_config_nodes() {
 
     # now make sure everything is fully up-to-date
     if ! time dnf -y upgrade \
-                  --exclude fuse,mercury,daos,daos-\*; then
+                  --exclude "$EXCLUDE_UPGRADE"; then
         dump_repos
         exit 1
     fi
 
-    if ! time dnf -y install "${PYTHON_MACROS_RPM[@]}" ; then
-        dump_repos
-        exit 1
+    if [ -f /etc/do-release ]; then
+        cat /etc/do-release
     fi
-
-    cat /etc/do-release
     cat /etc/os-release
 
     exit 0