CORCI-261: build: Global product versioning (#7) (#1017)

Add a VERSION file for all things that want to display a version
to be able to read from.

Add an scons target "release" that takes a RELEASE variable to create
a new release from.  This updates:
- the VERSION file
- the RPM daos.spec
and then commits the changes.

Get the version of the libdaos library to import from the VERSION file.

Once a PR with an updated VERSION file lands to master, a GitHub
action will create a release on GitHub which will include creating a
tag on master where the PR was landed.

Signed-off-by: Brian J. Murrell <brian.murrell@intel.com>

.github/actions/make_release/Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine:3.10
+
+COPY entrypoint.sh /entrypoint.sh
+
+RUN apk add --no-cache git curl
+
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/make_release/action.yml

@@ -0,0 +1,6 @@
+name: 'Make Release'
+description: 'Make a release when requested'
+author: 'Brian J. Murrell'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/make_release/entrypoint.sh

@@ -0,0 +1,42 @@
+#!/bin/sh -l
+
+# Only need to do any of this if the version has been updated
+# NOTE: The diff-index with HEAD^ implies that the VERSION
+#       must be updated in the last commit.  But version update
+#       PRs really should just be a single commit since we
+#       script the creation of the version update PR.
+if ! git diff-index --name-only HEAD^ | grep -q VERSION; then
+     echo "VERSION not updated, exiting"
+     exit 0
+fi
+
+# Don't create a release for the first time the VERSION file is
+# added as that point is most surely not an actual release.
+if git diff-tree --diff-filter=A --no-commit-id --name-status -r HEAD |
+   grep -q VERSION; then
+     echo "VERSION being added not updated, exiting"
+     exit 0
+fi
+
+release=$(cat VERSION)
+
+# Ensure that the GITHUB_TOKEN secret is included
+if [ -z "$GITHUB_TOKEN" ]; then
+  echo "Set the GITHUB_TOKEN env variable."
+  exit 1
+fi
+
+json="{
+  \"tag_name\": \"v$release\",
+  \"target_commitish\": \"$GITHUB_SHA\",
+  \"name\": \"Release $release\",
+  \"body\": \"DAOS release $release\",
+  \"draft\": false,
+  \"prerelease\": false
+}"
+
+curl --request POST \
+  --url https://api.github.com/repos/"${GITHUB_REPOSITORY}"/releases \
+  --header "Authorization: Bearer $GITHUB_TOKEN" \
+  --header 'Content-Type: application/json' \
+  --data "$json"

.github/workflows/crease_release.yml

@@ -0,0 +1,17 @@
+name: Create Release
+# This workflow is triggered on pushes to the master branch of the repository.
+on:
+  push:
+    branches:
+    - master
+
+jobs:
+  make_release:
+    name: Create Release
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - uses: ./.github/actions/make_release
+      id: make_release
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

SConstruct

@@ -2,6 +2,9 @@
 import sys
 import os
 import platform
+import subprocess
+import time
+import errno
 from SCons.Script import BUILD_TARGETS
 
 sys.path.insert(0, os.path.join(Dir('#').abspath, 'utils'))
@@ -16,7 +19,44 @@ DESIRED_FLAGS = ['-Wno-gnu-designator',
 PP_ONLY_FLAGS = ['-Wno-parentheses-equality', '-Wno-builtin-requires-header',
                  '-Wno-unused-function']
 
-DAOS_VERSION = "0.6.0"
+def get_version():
+    """ Read version from VERSION file """
+    with open("VERSION", "r") as version_file:
+        return version_file.read().rstrip()
+
+DAOS_VERSION = get_version()
+
+def update_rpm_version(version):
+    """ Update the version (and release) in the RPM specfile """
+    spec = open("utils/rpms/daos.spec", "r").readlines()
+    for line_num, line in enumerate(spec):
+        if line.startswith("Version:"):
+            spec[line_num] = "Version:       {}\n".format(version)
+        if line.startswith("Release:"):
+            spec[line_num] = "Release:       1%{?relval}%{?dist}\n"
+        if line == "%changelog\n":
+            try:
+                packager = subprocess.Popen(
+                    'rpmdev-packager', stdout=subprocess.PIPE).communicate(
+                    )[0].strip().decode('UTF-8')
+            except OSError:
+                print("You need to have the rpmdev-packager tool (from the "
+                      "rpmdevtools RPM on EL7) in order to make releases.\n\n"
+                      "Additionally, you should define %packager in "
+                      "~/.rpmmacros as such:\n"
+                      "%packager	John A. Doe <john.doe@intel.com>"
+                      "so that package changelog entries are well defined")
+                exit(1)
+            date_str = time.strftime('%a %b %d %Y', time.gmtime())
+            spec.insert(line_num + 1, "\n")
+            spec.insert(line_num + 1,
+                        "- Version bump up to {}\n".format(version))
+            spec.insert(line_num + 1,
+                        u'* {} {} - {}-1\n'.format(date_str,
+                                                   packager,
+                                                   version))
+            break
+    open("utils/rpms/daos.spec", "w").writelines(spec)
 
 def is_platform_arm():
     """Detect if platform is ARM"""
@@ -54,6 +94,147 @@ def preload_prereqs(prereqs):
     prereqs.load_definitions(prebuild=reqs)
 
 def scons():
+    if COMMAND_LINE_TARGETS == ['release']:
+        org_name = "daos-stack"
+        remote_name = "origin"
+        try:
+            import pygit2
+            import github
+            import yaml
+        except ImportError:
+            print("You need yaml, pygit2 and pygithub python modules to "
+                  "create releases")
+            exit(1)
+
+        try:
+            token = yaml.safe_load(open(os.path.join(os.path.expanduser("~"),
+                                                     ".config", "hub"), 'r')
+                                  )['github.com'][0]['oauth_token']
+        except IOError as excpn:
+            if excpn.errno == errno.ENOENT:
+                print("You need to install hub (from the hub RPM on EL7) to "
+                      "and run it at least once to create an authorization "
+                      "token in order to create releases")
+                exit(1)
+            raise
+
+        variables = Variables()
+        variables.Add('RELEASE', 'Set to the release version to make', None)
+        env = Environment(variables=variables)
+        try:
+            version = env['RELEASE']
+        except KeyError:
+            print ("Usage: scons RELEASE=x.y.z release")
+            exit(1)
+
+        # create a branch for the PR
+        branch = 'create-release-{}'.format(version)
+        print("Creating a branch for the PR...")
+        repo = pygit2.Repository('.git')
+        master = repo.lookup_reference(
+            'refs/remotes/{}/master'.format(remote_name))
+        try:
+            repo.branches.create(branch, repo[master.target])
+        except pygit2.AlreadyExistsError: # pylint: disable=no-member
+            print("Branch {} exists in GitHub already\n"
+                  "See https://github.com/{}/daos/branches".format(branch,
+                                                                   org_name))
+            exit(1)
+
+        # and check it out
+        print("Checking out branch for the PR...")
+        repo.checkout(repo.lookup_branch(branch))
+
+        print("Updating the VERSION file...")
+        with open("VERSION", "w") as version_file:
+            version_file.write(version + '\n')
+
+        print("Updating the RPM specfile...")
+        update_rpm_version(version)
+
+        print("Committing the changes...")
+        # now create the commit
+        index = repo.index
+        index.read()
+        author = repo.default_signature
+        committer = repo.default_signature
+        summary = "Update version to v{}".format(version)
+        # pylint: disable=no-member
+        message = "{}\n\n" \
+                  "Signed-off-by: {} <{}>".format(summary,
+                                                  repo.default_signature.name,
+                                                  repo.default_signature.email)
+        # pylint: enable=no-member
+        index.add("utils/rpms/daos.spec")
+        index.add("VERSION")
+        index.write()
+        tree = index.write_tree()
+        # pylint: disable=no-member
+        repo.create_commit('HEAD', author, committer, message, tree,
+                           [repo.head.target])
+        # pylint: enable=no-member
+
+        # set up authentication callback
+        class MyCallbacks(pygit2.RemoteCallbacks):
+            """ Callbacks for pygit2 """
+            def credentials(self, url, username_from_url, allowed_types): # pylint: disable=method-hidden
+                if allowed_types & pygit2.credentials.GIT_CREDTYPE_SSH_KEY:
+                    if "SSH_AUTH_SOCK" in os.environ:
+                        # Use ssh agent for authentication
+                        return pygit2.KeypairFromAgent(username_from_url)
+                    #else:
+                    # need to determine if key is passphrase protected and ask
+                    # for the passphrase in order to use this method
+                    #    ssh_key = os.path.join(os.path.expanduser("~"),
+                    #                           ".ssh", "id_rsa")
+                    #    return pygit2.Keypair("git", ssh_key + ".pub",
+                    #                          ssh_key, "")
+                #elif allowed_types & pygit2.credentials.GIT_CREDTYPE_USERNAME:
+                # this is not really useful in the GitHub context
+                #    return pygit2.Username("git")
+                else:
+                    raise Exception("No supported credential types allowed "
+                                    "by remote end.  SSH_AUTH_SOCK not found "
+                                    "in your environment.  Are you running an "
+                                    "ssh-agent?")
+
+        # and push it
+        print("Pushing the changes to GitHub...")
+        remote = repo.remotes[remote_name]
+        try:
+            remote.push(['refs/heads/{}'.format(branch)],
+                        callbacks=MyCallbacks())
+        except pygit2.GitError:
+            print("Error pushing branch.  Does it exist in GitHub already?\n"
+                  "See https://github.com/{}/daos/branches".format(org_name))
+            exit(1)
+
+        print("Creating the PR...")
+        # now create a PR for it
+        gh_context = github.Github(token)
+        try:
+            org = gh_context.get_organization(org_name)
+            repo = org.get_repo('daos')
+        except github.UnknownObjectException:
+            # maybe not an organization
+            repo = gh_context.get_repo('{}/daos'.format(org_name))
+        new_pr = repo.create_pull(title=summary, body="", base="master",
+                                  head="{}:{}".format(org_name, branch))
+
+        print("Successfully created PR#{0} for this version "
+              "update:\n"
+              "https://github.com/{1}/daos/pull/{0}/".format(new_pr.number,
+                                                             org_name))
+
+        print("Self-assigning the PR...")
+        # self-assign the PR
+        new_pr.as_issue().add_to_assignees(
+            gh_context.get_user(gh_context.get_user().login))
+
+        print("Done.")
+
+        exit(0)
+
     """Execute build"""
     if os.path.exists('scons_local'):
         try:
@@ -98,6 +279,7 @@ def scons():
     buildinfo.save('.build_vars.json')
     env.InstallAs("$PREFIX/TESTING/.build_vars.sh", ".build_vars.sh")
     env.InstallAs("$PREFIX/TESTING/.build_vars.json", ".build_vars.json")
+    env.InstallAs("$PREFIX/lib/daos/VERSION", "VERSION")
 
     env.Install('$PREFIX/etc', ['utils/memcheck-daos-client.supp'])
 

VERSION

@@ -0,0 +1 @@
+0.6.0

src/utils/py/daos_api.py

@@ -2108,13 +2108,19 @@ def kill(self, force):
                                .format(ret))
 
 
+# pylint: disable=pylint-too-few-public-methods
 class DaosContext(object):
     """Provides environment and other info for a DAOS client."""
 
     def __init__(self, path):
         """ setup the DAOS API and MPI """
 
-        self.libdaos = ctypes.CDLL(path+"libdaos.so.0.6.0",
+        # first find the DAOS version
+        with open(os.path.join(path, "daos", "VERSION"),
+                  "r") as version_file:
+            daos_version = version_file.read().rstrip()
+
+        self.libdaos = ctypes.CDLL(path+"libdaos.so.{}".format(daos_version),
                                    mode=ctypes.DEFAULT_MODE)
         ctypes.CDLL(path+"libdaos_common.so",
                     mode=ctypes.RTLD_GLOBAL)
@@ -2186,6 +2192,7 @@ def __del__(self):
     def get_function(self, function):
         """ call a function through the API """
         return self.ftable[function]
+# pylint: enable=pylint-too-few-public-methods
 
 
 class DaosLog:

utils/docker/Dockerfile.centos.7

@@ -58,9 +58,9 @@ RUN yum -y install epel-release; \
         libaio-devel libcmocka-devel libevent-devel libiscsi-devel     \
         libtool libtool-ltdl-devel libuuid-devel libyaml-devel         \
         make meson nasm ninja-build numactl-devel openssl-devel        \
-	pandoc patch pylint python python-devel python36-devel         \
-	python-pep8 python-requests python2-pygithub readline-devel    \
-	scons sg3_utils ShellCheck yasm
+        pandoc patch pylint python python-devel python36-devel         \
+        python-pep8 python-pygit2 python2-pygithub python-requests     \
+        readline-devel scons sg3_utils ShellCheck yasm
 
 # DAOS specific
 RUN yum -y install \

utils/rpms/daos.spec

@@ -172,6 +172,7 @@ install -m 644 utils/systemd/daos-agent.service %{?buildroot}/%{_unitdir}
 %{_libdir}/daos_srv/libplacement.so
 # Certificate generation files
 %{daoshome}/certgen/
+%{daoshome}/VERSION
 %doc
 
 %files server