diff --git a/libfabric.spec b/libfabric.spec index 6fc5c93..66c6566 100644 --- a/libfabric.spec +++ b/libfabric.spec @@ -4,6 +4,12 @@ %global bugrelease 0 %global dl_version %{major}.%{minor}.%{bugrelease}%{?prerelease:%{prerelease}} +%if (0%{?suse_version} > 0) +%global __debug_package 1 +%global _debuginfo_subpackages 0 +%debug_package +%endif + Name: libfabric Version: %{major}.%{minor}.%{bugrelease}%{?prerelease:~%{prerelease}} Release: 1%{?dist} diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 index 2514d84..55510f7 100644 --- a/packaging/Dockerfile.centos.7 +++ b/packaging/Dockerfile.centos.7 @@ -1,5 +1,5 @@ # -# Copyright 2018-2020, Intel Corporation +# Copyright 2018-2021, Intel Corporation # # 'recipe' for Docker to build an RPM # @@ -17,7 +17,7 @@ ARG UID=1000 # Install basic tools RUN yum install -y epel-release RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ - git python-srpm-macros + git python-srpm-macros dnf # Add build user (to keep rpmbuild happy) ENV USER build diff --git a/packaging/Dockerfile.coverity b/packaging/Dockerfile.coverity index 0715f4d..7eed2c3 100755 --- a/packaging/Dockerfile.coverity +++ b/packaging/Dockerfile.coverity @@ -1,5 +1,5 @@ # -# Copyright 2018-2019, Intel Corporation +# Copyright 2018-2020, Intel Corporation # # 'recipe' for Docker to build for a Coverity scan. # diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index a55b2be..3f6fe67 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,12 +1,12 @@ # -# Copyright 2018-2020, Intel Corporation +# Copyright 2018-2021, Intel Corporation # # 'recipe' for Docker to build an RPM # # Pull base image FROM fedora:latest -MAINTAINER daos-stack +LABEL maintainer="daos@daos.groups.io>" # use same UID as host and default value of 1000 if not specified ARG UID=1000 @@ -28,4 +28,6 @@ RUN usermod -a -G mock $USER RUN grep use_nspawn /etc/mock/site-defaults.cfg || \ echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg -RUN chmod g+w /etc/mock/* +ARG CACHEBUST +RUN dnf -y upgrade && \ + dnf clean all diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index cf16bce..f956145 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -1,11 +1,11 @@ # -# Copyright 2019-2020, Intel Corporation +# Copyright 2019-2021, Intel Corporation # # 'recipe' for Docker to build an Debian package # # Pull base image FROM ubuntu:20.04 -MAINTAINER daos-stack +LABEL org.opencontainers.image.authors="daos@daos.groups.io" # use same UID as host and default value of 1000 if not specified ARG UID=1000 @@ -14,10 +14,10 @@ ARG REPO_UBUNTU_20_04="" # Install basic tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - autoconf bash curl debhelper dh-make dpkg-dev doxygen gcc \ - git git-buildpackage locales make patch pbuilder rpm wget \ - ca-certificates scons python3-distutils pkg-config \ - python3-dev python3-distro + autoconf bash ca-certificates curl debhelper dh-make \ + dpkg-dev dh-python doxygen gcc git git-buildpackage locales \ + make patch pbuilder pkg-config python3-dev python3-distro \ + python3-distutils rpm scons wget # rpmdevtools RUN echo "deb [trusted=yes] ${REPO_URL}${REPO_UBUNTU_20_04} focal main" > /etc/apt/sources.list.d/daos-stack-ubuntu-stable-local.list diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index c126c81..44006e0 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -27,36 +27,38 @@ SPECTOOL := spectool # DISTRO_BASE (i.e. EL_7) # from the CHROOT_NAME ifeq ($(CHROOT_NAME),epel-7-x86_64) -DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) -VERSION_ID := 7 -DISTRO_ID := el7 -DISTRO_BASE := EL_7 -SED_EXPR := 1s/$(DIST)//p +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 7 +DISTRO_ID := el7 +DISTRO_BASE := EL_7 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 7 +SED_EXPR := 1s/$(DIST)//p endif ifeq ($(CHROOT_NAME),epel-8-x86_64) -DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) -VERSION_ID := 8 -DISTRO_ID := el8 -DISTRO_BASE := EL_8 -SED_EXPR := 1s/$(DIST)//p -endif -ifeq ($(CHROOT_NAME),opensuse-leap-15.1-x86_64) -VERSION_ID := 15.1 -DISTRO_ID := sl15.1 -DISTRO_BASE := LEAP_15 -SED_EXPR := 1p +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 8 +DISTRO_ID := el8 +DISTRO_BASE := EL_8 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 8 +SED_EXPR := 1s/$(DIST)//p endif ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64) -VERSION_ID := 15.2 -DISTRO_ID := sl15.2 -DISTRO_BASE := LEAP_15 -SED_EXPR := 1p -endif -ifeq ($(CHROOT_NAME),leap-42.3-x86_64) -# TBD if support is ever resurrected -endif -ifeq ($(CHROOT_NAME),sles-12.3-x86_64) -# TBD if support is ever resurrected +VERSION_ID := 15.2 +DISTRO_ID := sl15.2 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.2 +SED_EXPR := 1p +endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.3-x86_64) +VERSION_ID := 15.3 +DISTRO_ID := sl15.3 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.2 +SED_EXPR := 1p endif endif ifeq ($(ID),centos) diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 1963dbb..8470b7a 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -10,7 +10,9 @@ SHELL=/bin/bash -include Makefile.local # default to Leap 15 distro for chrootbuild +ifeq ($(MAKECMDGOALS),chrootbuild) CHROOT_NAME ?= opensuse-leap-15.2-x86_64 +endif include packaging/Makefile_distro_vars.mk ifeq ($(DEB_NAME),) @@ -28,6 +30,11 @@ RPM_BUILD_OPTIONS += $(EXTERNAL_RPM_BUILD_OPTIONS) # some defaults the caller can override PACKAGING_CHECK_DIR ?= ../packaging LOCAL_REPOS ?= true +ifeq ($(ID_LIKE),debian) +DAOS_REPO_TYPE ?= LOCAL +else +DAOS_REPO_TYPE ?= STABLE +endif TEST_PACKAGES ?= ${NAME} # unfortunately we cannot always name the repo the same as the project @@ -36,9 +43,24 @@ REPO_NAME ?= $(NAME) PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos: *\(.*\)/\1/p') LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') +EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') UBUNTU_20_04_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p') +ifneq ($(PKG_GIT_COMMIT),) +ifeq ($(GITHUB_PROJECT),) +ifeq ($(GIT_PROJECT),) +$(error You must set either GITHUB_PROJECT or GIT_PROJECT if you set PKG_GIT_COMMIT) +endif +endif +BUILD_DEFINES := --define "commit $(PKG_GIT_COMMIT)" +RPM_BUILD_OPTIONS := $(BUILD_DEFINES) +GIT_DIFF_EXCLUDES := $(PATCH_EXCLUDE_FILES:%=':!%') +endif + COMMON_RPM_ARGS := --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) +ifneq ($(CHROOT_NAME),) +COMMON_RPM_ARGS += --define "chroot_name $(CHROOT_NAME)" $(BUILD_DEFINES) +endif SPEC := $(shell if [ -f $(NAME)-$(DISTRO_BASE).spec ]; then echo $(NAME)-$(DISTRO_BASE).spec; else echo $(NAME).spec; fi) VERSION = $(eval VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p'))$(VERSION) DEB_RVERS := $(subst $(DOT),\$(DOT),$(VERSION)) @@ -285,25 +307,80 @@ debs: $(DEBS) ls: $(TARGETS) ls -ld $^ +ifneq ($(PKG_GIT_COMMIT),) +# This not really intended to run in CI. It's meant as a developer +# convenience to generate the needed patch and add it to the repo to +# be committed. +$(VERSION)..$(PKG_GIT_COMMIT).patch: +ifneq ($(GITHUB_PROJECT),) + # it really sucks that GitHub's "compare" returns such dirty patches + #curl -O 'https://github.com/$(GITHUB_PROJECT)/compare/$@' + git clone https://github.com/$(GITHUB_PROJECT).git +else + git clone $(GIT_PROJECT) +endif + set -x; pushd $(NAME) && \ + trap 'popd && rm -rf $(NAME)' EXIT; \ + echo git diff $(VERSION)..$(PKG_GIT_COMMIT) --stat -- \ + $(GIT_DIFF_EXCLUDES ); \ + git diff $(VERSION)..$(PKG_GIT_COMMIT) -- \ + $(GIT_DIFF_EXCLUDES) > ../$@; \ + popd; \ + trap 'rm -rf $(NAME)' EXIT; \ + git add $@ +patch: $(VERSION)..$(PKG_GIT_COMMIT).patch +else +patch: + echo "PKG_GIT_COMMIT is not defined" +endif + # *_LOCAL_* repos are locally built packages. # *_GROUP_* repos are a local mirror of a group of upstream repos. # *_GROUP_* repos may not supply a repomd.xml.key. ifeq ($(LOCAL_REPOS),true) -ifneq ($(REPOSITORY_URL),) -ifneq ($(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO),) -ifeq ($(ID_LIKE),debian) -# $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists -# of values with spaces as environment variables -$(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] -endif -$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS) $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO)/ -endif -$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)| -ifneq ($(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO),) -$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_GROUP_REPO)/| -endif -endif -endif + ifneq ($(REPOSITORY_URL),) + # group repos are not working in Nexus so we hack in the group members directly below + #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO),) + #DISTRO_REPOS = $(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO) + #$(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_DOCKER_$(DAOS_REPO_TYPE)_REPO)/ + #endif + ifneq ($(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO),) + ifeq ($(ID_LIKE),debian) + # $(DISTRO_BASE)_LOCAL_REPOS is a list separated by | because you cannot pass lists + # of values with spaces as environment variables + $(DISTRO_BASE)_LOCAL_REPOS := [trusted=yes] + else + $(DISTRO_BASE)_LOCAL_REPOS := $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_LOCAL_REPO) + DISTRO_REPOS = disabled # any non-empty value here works and is not used beyond testing if the value is empty or not + endif # ifeq ($(ID_LIKE),debian) + ifeq ($(DISTRO_BASE), EL_8) + # hack to use 8.3 non-group repos on EL_8 + $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/centos-8.3-base-x86_64-proxy|$(REPOSITORY_URL)repository/centos-8.3-extras-x86_64-proxy|$(REPOSITORY_URL)repository/epel-el-8-x86_64-proxy) + else ifeq ($(DISTRO_BASE), EL_7) + # hack to use 7.9 non-group repos on EL_7 + $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/centos-7.9-base-x86_64-proxy|$(REPOSITORY_URL)repository/centos-7.9-extras-x86_64-proxy|$(REPOSITORY_URL)repository/centos-7.9-updates-x86_64-proxy|$(REPOSITORY_URL)repository/epel-el-7-x86_64-proxy) + else ifeq ($(DISTRO_BASE), LEAP_15) + # hack to use 15 non-group repos on LEAP_15 + $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(subst $(ORIG_TARGET_VER),$(DISTRO_VERSION),$(REPOSITORY_URL)repository/opensuse-15.2-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-update-oss-x86_64-provo-mirror-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-update-non-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-non-oss-x86_64-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-repo-sle-update-proxy|$(REPOSITORY_URL)repository/opensuse-15.2-repo-backports-update-proxy) + else + # debian + $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS) $(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO) + endif # ifeq ($(DISTRO_BASE), *) + endif #ifneq ($(DAOS_STACK_$(DISTRO_BASE)_$(DAOS_REPO_TYPE)_REPO),) + ifneq ($(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO),) + $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_APPSTREAM_REPO) + endif + # group repos are not working in Nexus so we hack in the group members directly above + ifneq ($(DAOS_STACK_$(DISTRO_BASE)_POWERTOOLS_REPO),) + $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_$(DISTRO_BASE)_POWERTOOLS_REPO) + endif + ifneq ($(ID_LIKE),debian) + ifneq ($(DAOS_STACK_INTEL_ONEAPI_REPO),) + $(DISTRO_BASE)_LOCAL_REPOS := $($(DISTRO_BASE)_LOCAL_REPOS)|$(REPOSITORY_URL)$(DAOS_STACK_INTEL_ONEAPI_REPO) + endif # ifneq ($(DAOS_STACK_INTEL_ONEAPI_REPO),) + endif # ifneq ($(ID_LIKE),debian) + endif # ifneq ($(REPOSITORY_URL),) +endif # ifeq ($(LOCAL_REPOS),true) ifeq ($(ID_LIKE),debian) chrootbuild: $(DEB_TOP)/$(DEB_DSC) $(call distro_map) \ @@ -330,6 +407,7 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE) DISTRO_BASE_LOCAL_REPOS="$($(DISTRO_BASE)_LOCAL_REPOS)" \ MOCK_OPTIONS="$(MOCK_OPTIONS)" \ RPM_BUILD_OPTIONS='$(RPM_BUILD_OPTIONS)' \ + DISTRO_REPOS='$(DISTRO_REPOS)' \ TARGET="$<" \ packaging/rpm_chrootbuild endif @@ -377,7 +455,7 @@ endif test: # Test the rpmbuild by installing the built RPM $(call install_repos,$(REPO_NAME)@$(BRANCH_NAME):$(BUILD_NUMBER)) - yum -y install $(TEST_PACKAGES) + dnf -y install $(TEST_PACKAGES) show_spec: @echo '$(SPEC)' diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index e2b201d..8c1008e 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -4,44 +4,59 @@ set -uex # shellcheck disable=SC2153 IFS=\| read -r -a distro_base_local_repos <<< "$DISTRO_BASE_LOCAL_REPOS" +repo_adds=() +repo_dels=() -if [ -w /etc/mock/"$CHROOT_NAME".cfg ]; then - # Remove the distro repos - if [[ $CHROOT_NAME =~ opensuse-leap-* ]]; then - ed /etc/mock/"$CHROOT_NAME".cfg << EOF -/^# repos$/+2;/^"""$/-1d -wq -EOF - fi - echo -e "config_opts['yum.conf'] += \"\"\"\n" >> /etc/mock/"$CHROOT_NAME".cfg - for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do - branch="master" - build_number="lastSuccessfulBuild" - if [[ $repo = *@* ]]; then - branch="${repo#*@}" - repo="${repo%@*}" - if [[ $branch = *:* ]]; then - build_number="${branch#*:}" - branch="${branch%:*}" - fi +if [ -n "${REPOSITORY_URL:-}" ] && [ -n "$DISTRO_REPOS" ]; then + repo_dels+=("--disablerepo=\*") +fi + +: "${WORKSPACE:=$PWD}" +mock_config_dir="$WORKSPACE/mock" +original_cfg_file="/etc/mock/${CHROOT_NAME}.cfg" +cfg_file="$mock_config_dir/${CHROOT_NAME}.cfg" +mkdir -p "$mock_config_dir" +ln -sf /etc/mock/templates "$mock_config_dir/" +ln -sf /etc/mock/logging.ini "$mock_config_dir/" + +cp "$original_cfg_file" "$cfg_file" + +if [ "${CHROOT_NAME}" == "epel-8-x86_64" ]; then + echo -e "config_opts['module_enable'] = ['javapackages-tools:201801']" \ + >> "$cfg_file" +fi + +echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" +for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do + branch="master" + build_number="lastSuccessfulBuild" + if [[ $repo = *@* ]]; then + branch="${repo#*@}" + repo="${repo%@*}" + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" fi - echo -e "[$repo:$branch:$build_number]\n\ + fi + repo_adds+=("--enablerepo $repo:$branch:$build_number") + echo -e "[$repo:$branch:$build_number]\n\ name=$repo:$branch:$build_number\n\ baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ -gpgcheck=False\n" >> /etc/mock/"$CHROOT_NAME".cfg - done - for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do - repo_name=${repo##*://} - repo_name=${repo_name//\//_} - echo -e "[${repo_name//@/_}]\n\ +gpgcheck=False\n" >> "$cfg_file" +done +for repo in $JOB_REPOS "${distro_base_local_repos[@]}"; do + repo_name=${repo##*://} + repo_name=${repo_name//\//_} + repo_adds+=("--enablerepo $repo_name") + echo -e "[${repo_name//@/_}]\n\ name=${repo_name}\n\ baseurl=${repo}\n\ -enabled=1\n" >> /etc/mock/"$CHROOT_NAME".cfg - done - echo "\"\"\"" >> /etc/mock/"$CHROOT_NAME".cfg -else - echo "Unable to update /etc/mock/$CHROOT_NAME.cfg." - echo "You need to make sure it has the needed repos in it yourself." -fi -eval mock -r "$CHROOT_NAME" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" +enabled=1\n" >> "$cfg_file" +done +echo "\"\"\"" >> "$cfg_file" + +# shellcheck disable=SC2086 +eval mock --configdir "$mock_config_dir" -r "${CHROOT_NAME}" \ + ${repo_dels[*]} ${repo_adds[*]} \ + $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET"