BUILD: update azure-pipelines.yml for releasing

azure-pipelines.yml

@@ -1,10 +1,12 @@
 trigger:
-  - master
   - develop
+  - dev/*
+  - release/*
 
 pr:
-  - master
   - develop
+  - dev/*
+  - release/*
 
 # set the build name
 name: $[ variables['branchName'] ]
@@ -35,7 +37,8 @@ variables:
     branchName: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ]
   ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}:
     branchName: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ]
-  master_or_release: $[ or(startsWith(variables['branchName'], 'release'), eq(variables['branchName'], 'master')) ]
+  source_is_release_branch: $[ startsWith(variables['branchName'], 'release') ]
+  source_is_develop_branch: $[ or(startsWith(variables['branchName'], 'develop'), startsWith(variables['branchName'], 'dev/')) ]
   is_scheduled: $[ eq(variables['Build.Reason'], 'Schedule') ]
 
 stages:
@@ -117,7 +120,7 @@ stages:
     jobs:
     - job:
       displayName: 'pytest @ develop environment'
-      condition: ne(variables.master_or_release, 'True')
+      condition: ne(variables.source_is_release_branch, 'True')
 
       pool:
           vmImage: 'ubuntu-latest'
@@ -138,11 +141,11 @@ stages:
           inputs:
             targetType: 'inline'
             script: |
-              set -e
+              set -eux
               eval "$(conda shell.bash hook)"
               cd $(System.DefaultWorkingDirectory)/facet/
               export PYTHONPATH=$(System.DefaultWorkingDirectory)/facet/src/
-
+              export RUN_PACKAGE_VERSION_TEST=facet
               ENV_YML=$(Agent.TempDirectory)/environment-stripped.yml
               grep -vE 'gamma-pytools|sklearndf' environment.yml > "$ENV_YML"
               conda env create -f "$ENV_YML"
@@ -179,7 +182,7 @@ stages:
   # - for release branch: 3 build tests
   # - otherwise: 1 build test
   - stage: conda_tox_build
-    displayName: 'Conda build & test'
+    displayName: 'Conda/tox build & test'
     dependsOn: 'detect_build_config_changes'
     variables:
       conda_build_config_changed: $[ stageDependencies.detect_build_config_changes.checkout_and_diff.outputs['diff.conda_build_config_changed'] ]
@@ -189,7 +192,8 @@ stages:
       displayName: 'single'
       condition: >
         and(
-        ne(variables.master_or_release, 'True'),
+        ne(variables.source_is_release_branch, 'True'),
+        ne(variables.source_is_develop_branch, 'True'),
         ne(variables.is_scheduled, 'True'),
         ne(stageDependencies.detect_build_config_changes.checkout_and_diff.outputs['diff.conda_build_config_changed'], '0')
         )
@@ -238,7 +242,7 @@ stages:
             targetType: 'inline'
             script: |
               if [ "$BUILD_SYSTEM" = "conda" ] ; then eval "$(conda shell.bash hook)" ; fi
-
+              export RUN_PACKAGE_VERSION_TEST=facet
               cd $(Build.SourcesDirectory)/pytools
               ./make.py pytools $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
 
@@ -264,7 +268,8 @@ stages:
       displayName: 'matrix'
       condition: >
         or(
-        eq(variables.master_or_release, 'True'),
+        eq(variables.source_is_develop_branch, 'True'),
+        eq(variables.source_is_release_branch, 'True'),
         eq(variables.is_scheduled, 'True')
         )
 
@@ -328,8 +333,9 @@ stages:
           inputs:
             targetType: 'inline'
             script: |
-              set -e
+              set -eux
               if [ "$BUILD_SYSTEM" = "conda" ] ; then eval "$(conda shell.bash hook)" ; fi
+              export RUN_PACKAGE_VERSION_TEST=facet
               cd $(Build.SourcesDirectory)/pytools
               ./make.py pytools $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
               cd $(Build.SourcesDirectory)/sklearndf
@@ -363,7 +369,7 @@ stages:
             inputs:
               targetType: 'inline'
               script: |
-                set -e
+                set -eux
                 eval "$(conda shell.bash hook)"
                 cd $(System.DefaultWorkingDirectory)
                 mkdir static_scan
@@ -384,18 +390,83 @@ stages:
               importResults: false
               failBuildOnPolicyFail: false
 
-  # release on merge from release branch to master:
+  - stage: check_release
+    displayName: 'Check Release Version'
+    dependsOn: conda_tox_build
+
+    jobs:
+      - job:
+        displayName: 'Check Release'
+        condition: >
+          or(
+          eq(variables.source_is_release_branch, 'True'),
+          eq(variables.source_is_develop_branch, 'True')
+          )
+
+        steps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: '3.7.*'
+            displayName: 'use Python 3.7'
+
+          - checkout: self
+          - checkout: pytools
+
+          - task: Bash@3
+            env:
+              BRANCH_NAME: $(branchName)
+            inputs:
+              targetType: inline
+              condition: ne(variables.branchName, 'develop')
+              script: |
+                set -eux
+                python -m pip install "toml==0.10.*"
+                cd $(System.DefaultWorkingDirectory)/pytools
+                python <<EOF
+                from os import environ
+                from make import ToxBuilder
+
+                branch_env = "BRANCH_NAME"
+                assert branch_env in environ, f"ENV variable '{branch_env}' with git branch name missing."
+                branch = environ[branch_env]
+                print(f"Checking package version consistency with branch: {branch}")
+                if branch == "develop":
+                  exit(0)
+                assert (branch.startswith("release/") or branch.startswith("dev/")
+                ), "This check should only run on versioned branches – check pipeline."
+
+                branch_version = branch.split("/", maxsplit=1)[1]
+                package_version = ToxBuilder("facet", "default").package_version
+
+                assert (
+                    package_version == branch_version
+                ), f"Package version '{package_version}' does not match '{branch_version}' from branch."
+
+                print("Check passed.")
+
+                EOF
+
+            displayName: "Check version consistency"
+
+
+  # release on merges into release/*:
   # - add release tag
   # - create GitHub release with changelog
   # - upload conda and pip packages as artifacts to GitHub
   - stage:
     displayName: 'Release'
-    dependsOn: conda_tox_build
+    dependsOn: check_release
+    variables:
+    - group: artifact_publication
 
     jobs:
       - job:
         displayName: 'Release'
-        condition: eq(replace(variables['Build.SourceBranch'], 'refs/heads/', ''), 'master')
+        condition: >
+          or(
+          eq(variables.source_is_release_branch, 'True'),
+          eq(variables.source_is_develop_branch, 'True')
+          )
 
         steps:
           - task: UsePythonVersion@0
@@ -411,7 +482,7 @@ stages:
             inputs:
               targetType: inline
               script: |
-                set -e
+                set -eux
                 echo "Getting version"
                 pip install packaging
                 cd $(System.DefaultWorkingDirectory)/facet/src
@@ -433,7 +504,45 @@ stages:
             inputs:
               artifactName: conda_default
 
+          - script: |
+              set -eux
+              cd $(System.DefaultWorkingDirectory)
+              eval "$(conda shell.bash hook)"
+              conda install -y anaconda-client
+              anaconda login --username "${CONDA_USERNAME}" --password "${CONDA_PASSWORD}"
+              anaconda upload --user BCG_Gamma --force $(System.ArtifactsDirectory)/conda_default/conda/noarch/gamma-facet-*.tar.bz2
+              anaconda logout
+              echo "##vso[task.setvariable variable=conda_published]True"
+            displayName: 'Publish to Anaconda'
+
+            condition: eq(variables['source_is_release_branch'], 'True')
+            env:
+              CONDA_PASSWORD: $(anaconda_pw)
+              CONDA_USERNAME: $(anaconda_user)
+
+          - script: |
+              set -eux
+              cd $(System.DefaultWorkingDirectory)/facet
+              pip install flit
+              flit install -s
+              flit publish
+              echo "##vso[task.setvariable variable=pypi_published]True"
+            displayName: 'Publish to PyPi'
+            condition: eq(variables['source_is_release_branch'], 'True')
+            env:
+              FLIT_PASSWORD: $(pypi_pw)
+              FLIT_USERNAME: $(pypi_user)
+
           - task: GitHubRelease@1
+            condition: >
+              and(
+                eq(variables['source_is_release_branch'], 'True'),
+                succeededOrFailed(),
+                or(
+                  eq(variables['conda_published'], 'True'),
+                  eq(variables['pypi_published'], 'True')
+                )
+              )
             inputs:
               gitHubConnection: github_release
               repositoryName: $(Build.Repository.Name)
@@ -470,7 +579,12 @@ stages:
     jobs:
     - job:
       displayName: 'Build and publish docs'
-      condition: eq(variables.master_or_release, 'True')
+      condition: >
+        or(
+        eq(variables.source_is_release_branch, 'True'),
+        eq(variables.source_is_develop_branch, 'True')
+        )
+
 
       pool:
           vmImage: 'ubuntu-latest'
@@ -498,7 +612,7 @@ stages:
           inputs:
             targetType: 'inline'
             script: |
-              set -e
+              set -eux
               cd $(System.DefaultWorkingDirectory)/facet
               git checkout --track origin/github-pages
               mkdir -p docs
@@ -513,7 +627,7 @@ stages:
           inputs:
             targetType: 'inline'
             script: |
-              set -e
+              set -eux
               eval "$(conda shell.bash hook)"
               cd $(System.DefaultWorkingDirectory)/facet
               echo "Checking out $(branchName)"
@@ -537,7 +651,7 @@ stages:
           inputs:
             targetType: 'inline'
             script: |
-              set -e
+              set -eux
               eval "$(conda shell.bash hook)"
               cp -r $(Build.ArtifactStagingDirectory)/old_docs/docs .
               echo "Current docs contents:"
@@ -553,10 +667,11 @@ stages:
           displayName: 'Update saved docs'
 
         - task: Bash@3
+          condition: eq(variables['source_is_release_branch'], 'True')
           inputs:
             targetType: 'inline'
             script: |
-              set -e
+              set -eux
               cd $(System.DefaultWorkingDirectory)/facet
               echo "Adjusting git credentials"
               git config --global user.name "Azure Pipelines"

pyproject.toml

@@ -22,7 +22,7 @@ requires = [
     "pandas         >=0.24,<1.3",
     "scipy          >=1.2,<1.6",
     "shap           >=0.34,<0.39",
-    "sklearndf      >=1.0.1,<2",
+    "sklearndf      >=1.0.2rc0,<2",
     # additional requirements of sklearndf
     "boruta         >=0.3",
     "lightgbm       >=3.0",
@@ -86,7 +86,7 @@ pandas         = "=0.24.*"
 python         = "=3.6.*"
 scipy          = "=1.2.*"
 shap           = "=0.34.*"
-sklearndf      = "<1.1"
+sklearndf      = "=1.0.2rc0"
 # additional requirements of sklearndf
 boruta         = "=0.3.*"
 lightgbm       = "=3.0.*"
@@ -104,7 +104,7 @@ pandas         = "<1.3"
 python         = "=3.8.*"
 scipy          = "<1.6"
 shap           = ">=0.37,<0.39"
-sklearndf      = "<2"
+sklearndf      = ">=1.0.2rc0,<2"
 # additional requirements of sklearndf
 boruta         = ">=0.3"
 lightgbm       = ">=3.0"

src/facet/__init__.py

@@ -6,7 +6,7 @@
 """
 
 
-__version__ = "1.0.2"
+__version__ = "1.0.2rc0"
 
 __logo__ = (
     r"""

test/test/test_package_version.py

@@ -0,0 +1,61 @@
+import logging
+import re
+from os import environ
+from urllib import request
+from xml.etree import ElementTree
+
+from pytest import mark
+
+import facet
+
+log = logging.getLogger(__name__)
+
+ENV_RUN_PACKAGE_VERSION_TEST = "RUN_PACKAGE_VERSION_TEST"
+
+
+@mark.skipif(
+    condition=environ.get(ENV_RUN_PACKAGE_VERSION_TEST, "") != facet.__name__,
+    reason="Parent build is not primarily for facet.",
+)
+def test_package_version() -> None:
+    dev_version = facet.__version__
+
+    log.info(f"Test package version – version set to: {dev_version}")
+    assert re.match(
+        r"^(\d)+\.(\d)+\.(\d)+(rc\d+)?$", dev_version
+    ), "facet.__version__ is not in MAJOR.MINOR.PATCH[rcN] format."
+
+    releases_uri = "https://pypi.org/rss/project/gamma-facet/releases.xml"
+
+    with request.urlopen(releases_uri) as response:
+        assert response.getcode() == 200, "Error getting releases from PyPi"
+        releases_xml = response.read()
+
+    tree = ElementTree.fromstring(releases_xml)
+    releases_nodes = tree.findall(path=".//channel//item//title")
+    releases = [r.text for r in releases_nodes]
+
+    log.info(f"Found these releases on PyPi:{releases}")
+
+    assert (
+        dev_version not in releases
+    ), f"Current package version {dev_version} already on PyPi"
+
+    is_minor_or_major_release = dev_version.endswith(".0")
+
+    if is_minor_or_major_release:
+        pre_releases = [
+            version
+            for version in releases
+            if re.match(f"{dev_version}rc\\d+$", version)
+        ]
+
+        assert pre_releases, (
+            f"Release of major or minor version {dev_version} "
+            f"requires at least one pre-release, e.g. {dev_version}rc0"
+        )
+
+        log.info(
+            f"Pre-release(s) {pre_releases} exist(s) – "
+            f"release of major/minor version {dev_version} allowed"
+        )