From 1f7d60d4b7c78dbcd948a35c5e1809df1016e2dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Schneider?= Date: Fri, 5 Mar 2021 13:21:06 +0100 Subject: [PATCH 1/4] BUILD: update azure-pipelines.yml for releasing --- azure-pipelines.yml | 157 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 136 insertions(+), 21 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d3c291b56..c59a5e6ff 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,10 +1,12 @@ trigger: - - master - develop + - dev/* + - release/* pr: - - master - develop + - dev/* + - release/* # set the build name name: $[ variables['branchName'] ] @@ -35,7 +37,8 @@ variables: branchName: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: branchName: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ] - master_or_release: $[ or(startsWith(variables['branchName'], 'release'), eq(variables['branchName'], 'master')) ] + source_is_release_branch: $[ startsWith(variables['branchName'], 'release') ] + source_is_develop_branch: $[ or(startsWith(variables['branchName'], 'develop'), startsWith(variables['branchName'], 'dev/')) ] is_scheduled: $[ eq(variables['Build.Reason'], 'Schedule') ] stages: @@ -117,7 +120,7 @@ stages: jobs: - job: displayName: 'pytest @ develop environment' - condition: ne(variables.master_or_release, 'True') + condition: ne(variables.source_is_release_branch, 'True') pool: vmImage: 'ubuntu-latest' @@ -138,11 +141,11 @@ stages: inputs: targetType: 'inline' script: | - set -e + set -eux eval "$(conda shell.bash hook)" cd $(System.DefaultWorkingDirectory)/facet/ export PYTHONPATH=$(System.DefaultWorkingDirectory)/facet/src/ - + export RUN_PACKAGE_VERSION_TEST=facet ENV_YML=$(Agent.TempDirectory)/environment-stripped.yml grep -vE 'gamma-pytools|sklearndf' environment.yml > "$ENV_YML" conda env create -f "$ENV_YML" @@ -179,7 +182,7 @@ stages: # - for release branch: 3 build tests # - otherwise: 1 build test - stage: conda_tox_build - displayName: 'Conda build & test' + displayName: 'Conda/tox build & test' dependsOn: 'detect_build_config_changes' variables: conda_build_config_changed: $[ stageDependencies.detect_build_config_changes.checkout_and_diff.outputs['diff.conda_build_config_changed'] ] @@ -189,7 +192,8 @@ stages: displayName: 'single' condition: > and( - ne(variables.master_or_release, 'True'), + ne(variables.source_is_release_branch, 'True'), + ne(variables.source_is_develop_branch, 'True'), ne(variables.is_scheduled, 'True'), ne(stageDependencies.detect_build_config_changes.checkout_and_diff.outputs['diff.conda_build_config_changed'], '0') ) @@ -238,7 +242,7 @@ stages: targetType: 'inline' script: | if [ "$BUILD_SYSTEM" = "conda" ] ; then eval "$(conda shell.bash hook)" ; fi - + export RUN_PACKAGE_VERSION_TEST=facet cd $(Build.SourcesDirectory)/pytools ./make.py pytools $(BUILD_SYSTEM) $(PKG_DEPENDENCIES) @@ -264,7 +268,8 @@ stages: displayName: 'matrix' condition: > or( - eq(variables.master_or_release, 'True'), + eq(variables.source_is_develop_branch, 'True'), + eq(variables.source_is_release_branch, 'True'), eq(variables.is_scheduled, 'True') ) @@ -328,8 +333,9 @@ stages: inputs: targetType: 'inline' script: | - set -e + set -eux if [ "$BUILD_SYSTEM" = "conda" ] ; then eval "$(conda shell.bash hook)" ; fi + export RUN_PACKAGE_VERSION_TEST=facet cd $(Build.SourcesDirectory)/pytools ./make.py pytools $(BUILD_SYSTEM) $(PKG_DEPENDENCIES) cd $(Build.SourcesDirectory)/sklearndf @@ -363,7 +369,7 @@ stages: inputs: targetType: 'inline' script: | - set -e + set -eux eval "$(conda shell.bash hook)" cd $(System.DefaultWorkingDirectory) mkdir static_scan @@ -384,18 +390,83 @@ stages: importResults: false failBuildOnPolicyFail: false - # release on merge from release branch to master: + - stage: check_release + displayName: 'Check Release Version' + dependsOn: conda_tox_build + + jobs: + - job: + displayName: 'Check Release' + condition: > + or( + eq(variables.source_is_release_branch, 'True'), + eq(variables.source_is_develop_branch, 'True') + ) + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7.*' + displayName: 'use Python 3.7' + + - checkout: self + - checkout: pytools + + - task: Bash@3 + env: + BRANCH_NAME: $(branchName) + inputs: + targetType: inline + condition: ne(variables.branchName, 'develop') + script: | + set -eux + python -m pip install "toml==0.10.*" + cd $(System.DefaultWorkingDirectory)/pytools + python < + or( + eq(variables.source_is_release_branch, 'True'), + eq(variables.source_is_develop_branch, 'True') + ) steps: - task: UsePythonVersion@0 @@ -411,7 +482,7 @@ stages: inputs: targetType: inline script: | - set -e + set -eux echo "Getting version" pip install packaging cd $(System.DefaultWorkingDirectory)/facet/src @@ -433,7 +504,45 @@ stages: inputs: artifactName: conda_default + - script: | + set -eux + cd $(System.DefaultWorkingDirectory) + eval "$(conda shell.bash hook)" + conda install -y anaconda-client + anaconda login --username "${CONDA_USERNAME}" --password "${CONDA_PASSWORD}" + anaconda upload --user BCG_Gamma --force $(System.ArtifactsDirectory)/conda_default/conda/noarch/gamma-facet-*.tar.bz2 + anaconda logout + echo "##vso[task.setvariable variable=conda_published]True" + displayName: 'Publish to Anaconda' + + condition: eq(variables['source_is_release_branch'], 'True') + env: + CONDA_PASSWORD: $(anaconda_pw) + CONDA_USERNAME: $(anaconda_user) + + - script: | + set -eux + cd $(System.DefaultWorkingDirectory)/facet + pip install flit + flit install -s + flit publish + echo "##vso[task.setvariable variable=pypi_published]True" + displayName: 'Publish to PyPi' + condition: eq(variables['source_is_release_branch'], 'True') + env: + FLIT_PASSWORD: $(pypi_pw) + FLIT_USERNAME: $(pypi_user) + - task: GitHubRelease@1 + condition: > + and( + eq(variables['source_is_release_branch'], 'True'), + succeededOrFailed(), + or( + eq(variables['conda_published'], 'True'), + eq(variables['pypi_published'], 'True') + ) + ) inputs: gitHubConnection: github_release repositoryName: $(Build.Repository.Name) @@ -470,7 +579,12 @@ stages: jobs: - job: displayName: 'Build and publish docs' - condition: eq(variables.master_or_release, 'True') + condition: > + or( + eq(variables.source_is_release_branch, 'True'), + eq(variables.source_is_develop_branch, 'True') + ) + pool: vmImage: 'ubuntu-latest' @@ -498,7 +612,7 @@ stages: inputs: targetType: 'inline' script: | - set -e + set -eux cd $(System.DefaultWorkingDirectory)/facet git checkout --track origin/github-pages mkdir -p docs @@ -513,7 +627,7 @@ stages: inputs: targetType: 'inline' script: | - set -e + set -eux eval "$(conda shell.bash hook)" cd $(System.DefaultWorkingDirectory)/facet echo "Checking out $(branchName)" @@ -537,7 +651,7 @@ stages: inputs: targetType: 'inline' script: | - set -e + set -eux eval "$(conda shell.bash hook)" cp -r $(Build.ArtifactStagingDirectory)/old_docs/docs . echo "Current docs contents:" @@ -553,10 +667,11 @@ stages: displayName: 'Update saved docs' - task: Bash@3 + condition: eq(variables['source_is_release_branch'], 'True') inputs: targetType: 'inline' script: | - set -e + set -eux cd $(System.DefaultWorkingDirectory)/facet echo "Adjusting git credentials" git config --global user.name "Azure Pipelines" From b3ce1dfafb7d81f8cc27005fbbf72395798f267e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Schneider?= Date: Thu, 11 Mar 2021 20:04:45 +0100 Subject: [PATCH 2/4] add test_package_version.py for gamma-facet --- test/test/test_package_version.py | 61 +++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 test/test/test_package_version.py diff --git a/test/test/test_package_version.py b/test/test/test_package_version.py new file mode 100644 index 000000000..c08815c4b --- /dev/null +++ b/test/test/test_package_version.py @@ -0,0 +1,61 @@ +import logging +import re +from os import environ +from urllib import request +from xml.etree import ElementTree + +from pytest import mark + +import facet + +log = logging.getLogger(__name__) + +ENV_RUN_PACKAGE_VERSION_TEST = "RUN_PACKAGE_VERSION_TEST" + + +@mark.skipif( + condition=environ.get(ENV_RUN_PACKAGE_VERSION_TEST, "") != facet.__name__, + reason="Parent build is not primarily for facet.", +) +def test_package_version() -> None: + dev_version = facet.__version__ + + log.info(f"Test package version – version set to: {dev_version}") + assert re.match( + r"^(\d)+\.(\d)+\.(\d)+(rc\d+)?$", dev_version + ), "facet.__version__ is not in MAJOR.MINOR.PATCH[rcN] format." + + releases_uri = "https://pypi.org/rss/project/gamma-facet/releases.xml" + + with request.urlopen(releases_uri) as response: + assert response.getcode() == 200, "Error getting releases from PyPi" + releases_xml = response.read() + + tree = ElementTree.fromstring(releases_xml) + releases_nodes = tree.findall(path=".//channel//item//title") + releases = [r.text for r in releases_nodes] + + log.info(f"Found these releases on PyPi:{releases}") + + assert ( + dev_version not in releases + ), f"Current package version {dev_version} already on PyPi" + + is_minor_or_major_release = dev_version.endswith(".0") + + if is_minor_or_major_release: + pre_releases = [ + version + for version in releases + if re.match(f"{dev_version}rc\\d+$", version) + ] + + assert pre_releases, ( + f"Release of major or minor version {dev_version} " + f"requires at least one pre-release, e.g. {dev_version}rc0" + ) + + log.info( + f"Pre-release(s) {pre_releases} exist(s) – " + f"release of major/minor version {dev_version} allowed" + ) From 64ee59c8d21377fcb9109522b9f34c06592c10c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Schneider?= Date: Thu, 11 Mar 2021 20:07:09 +0100 Subject: [PATCH 3/4] set version of gamma-facet to 1.0.2rc0 --- src/facet/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/facet/__init__.py b/src/facet/__init__.py index cbf1767a9..0cbaa088b 100644 --- a/src/facet/__init__.py +++ b/src/facet/__init__.py @@ -6,7 +6,7 @@ """ -__version__ = "1.0.2" +__version__ = "1.0.2rc0" __logo__ = ( r""" From 70be0c040a8baf8f5efdfbdcef8886428067c9da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Schneider?= Date: Thu, 11 Mar 2021 20:53:54 +0100 Subject: [PATCH 4/4] pyproject.toml: need to depend on sklearndf 1.0.2rc0 to use numpy >1.19 --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f3b708159..d409a111c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ requires = [ "pandas >=0.24,<1.3", "scipy >=1.2,<1.6", "shap >=0.34,<0.39", - "sklearndf >=1.0.1,<2", + "sklearndf >=1.0.2rc0,<2", # additional requirements of sklearndf "boruta >=0.3", "lightgbm >=3.0", @@ -86,7 +86,7 @@ pandas = "=0.24.*" python = "=3.6.*" scipy = "=1.2.*" shap = "=0.34.*" -sklearndf = "<1.1" +sklearndf = "=1.0.2rc0" # additional requirements of sklearndf boruta = "=0.3.*" lightgbm = "=3.0.*" @@ -104,7 +104,7 @@ pandas = "<1.3" python = "=3.8.*" scipy = "<1.6" shap = ">=0.37,<0.39" -sklearndf = "<2" +sklearndf = ">=1.0.2rc0,<2" # additional requirements of sklearndf boruta = ">=0.3" lightgbm = ">=3.0"