update conditional build logic depending on target branch and scheduling

azure-pipelines.yml

@@ -1,3 +1,6 @@
+# set the build name
+name: $[ variables['branchName'] ]
+
 # RUN TESTS AND FULL CONDA BUILD MATRIX EVERY NIGHT AT 4AM
 schedules:
 - cron: "0 4 * * *"
@@ -19,16 +22,63 @@ resources:
       name: BCG-Gamma/pytools
       ref: develop # todo - update to stable release
 
+variables:
+  ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')) }}:
+    branchName: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ]
+  ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}:
+    branchName: $[ replace(variables['System.PullRequest.TargetBranch'], 'refs/heads/', '') ]
+  master_or_release: >
+    ${{ or(
+    startsWith(variables['branchName'], 'release'),
+    eq(variables['branchName'], 'master'),
+    eq(variables['Build.Reason'], 'Schedule')
+    )
+    }}
+
 stages:
-  - stage: 'Quick_env_test'
-    displayName: 'Environment build & pytest'
+  - stage: detect_conda_changes
+    displayName: 'detect conda changes'
+
+    jobs:
+
+      - job: checkout_and_diff
+        steps:
+          - checkout: self
+
+          - task: Bash@3
+            name: diff
+            inputs:
+              targetType: 'inline'
+              script: |
+                cd $(Build.Repository.LocalPath)
+                git diff-index --exit-code "origin/$(branchName)" conda-build
+                build_changed="$?"
+                if [ "$build_changed" = 1 ]; then
+                echo "conda build config has been changed"
+                else
+                echo "conda build config is unchanged"
+                fi
+                echo "##vso[task.setvariable variable=conda_build_config_changed;isOutput=true]$build_changed"
+
+
+  - stage:
+    displayName: 'simple pytest'
+    dependsOn: 'detect_conda_changes'
+    variables:
+      conda_build_config_changed: $[ stageDependencies.detect_conda_changes.checkout_and_diff.outputs['diff.conda_build_config_changed'] ]
 
     jobs:
     - job:
-      displayName: 'Setup & testing'
+      displayName: 'pytest @ develop environment'
+      condition: >
+        and(
+        ne(variables.master_or_release, 'True'),
+        eq(stageDependencies.detect_conda_changes.checkout_and_diff.outputs['diff.conda_build_config_changed'], '0')
+        )
 
       pool:
           vmImage: 'ubuntu-latest'
+
       steps:
         - task: UsePythonVersion@0
           inputs:
@@ -38,6 +88,7 @@ stages:
         - checkout: self
         - checkout: sklearndf
         - checkout: pytools
+
         - script: dir $(Build.SourcesDirectory)
 
         - task: Bash@3
@@ -58,7 +109,7 @@ stages:
               coverage run -m pytest test/test/
               coverage xml
               coverage html
-          displayName: 'Pytest'
+          displayName: 'pytest'
 
         - task: PublishTestResults@2
           condition: succeededOrFailed()
@@ -77,29 +128,37 @@ stages:
   # TESTING MATRIX OF PYTHON & SKLEARN VERSIONS
   # - FOR RELEASE BRANCH: 3 BUILD TESTS
   # - OTHERWISE: 1 BUILD TEST
-  - stage: 'Conda_build_test'
-    dependsOn:
-      - 'Quick_env_test'
-    displayName: 'Test conda environment builds'
+  - stage:
+    displayName: 'conda build & pytest'
+    dependsOn: 'detect_conda_changes'
+    variables:
+      conda_build_config_changed: $[ stageDependencies.detect_conda_changes.checkout_and_diff.outputs['diff.conda_build_config_changed'] ]
 
     jobs:
     - job:
-      displayName: 'Setup & testing: Single build'
-      condition: |
-        not(or(contains(variables['Build.SourceBranch'], 'refs/heads/release'),
-        eq(variables['Build.SourceBranch'], 'refs/heads/master'),
-        eq(variables['Build.Reason'], 'Schedule')))
+      displayName: '(single)'
+      condition: >
+        and(
+        ne(variables.master_or_release, 'True'),
+        ne(stageDependencies.detect_conda_changes.checkout_and_diff.outputs['diff.conda_build_config_changed'], '0')
+        )
 
       pool:
           vmImage: 'ubuntu-latest'
       strategy:
         matrix:
           maximum_dependencies:
-            FACET_V_PYTHON_BUILD: '3.8'
-            FACET_V_SKLEARN: '==0.23'
-            FACET_V_PANDAS: '==1.0.0'
-            FACET_V_NUMPY: '=>1.16'
-            FACET_V_SHAP: '==0.35'
+            FACET_V_PYTHON_BUILD: '=3.8.*'
+            FACET_V_PYTHON: '=3.8.5'
+            FACET_V_PANDAS: '=1.1.*'
+            FACET_V_NUMPY: '=1.19.*'
+            FACET_V_SCIPY: '=1.5.*'
+            FACET_V_MATPLOT: '=3.3.*'
+            FACET_V_SKLEARN: '>=0.23.1,<0.24'
+            FACET_V_BORUTA: '=0.3.*'
+            FACET_V_LGBM: '=3.0.*'
+            FACET_V_JOBLIB: '=0.16.*'
+            FACET_V_SHAP: '=0.35'
 
       steps:
         - task: UsePythonVersion@0
@@ -110,61 +169,79 @@ stages:
         - checkout: self
         - checkout: sklearndf
         - checkout: pytools
+
         - script: dir $(Build.SourcesDirectory)
 
         - task: Bash@3
           inputs:
             targetType: 'inline'
             script: |
               eval "$(conda shell.bash hook)"
+
               export PYTHONPATH=$(System.DefaultWorkingDirectory)/facet/src/
               export FACET_V_PYTHON_BUILD=$(FACET_V_PYTHON_BUILD)
               export FACET_V_PANDAS=$(FACET_V_PANDAS)
               export FACET_V_SKLEARN=$(FACET_V_SKLEARN)
               export FACET_V_JOBLIB=$(FACET_V_JOBLIB)
               export FACET_V_NUMPY=$(FACET_V_NUMPY)
               export FACET_V_SHAP=$(FACET_V_SHAP)
-              conda activate facet-develop
+
               conda install -y conda-build
+
               cd $(System.DefaultWorkingDirectory)/pytools/
               make package
+
               cd $(System.DefaultWorkingDirectory)/sklearndf/
               make package
+
               cd $(System.DefaultWorkingDirectory)/facet/
               make package
           displayName: "Build conda package"
 
     - job:
-      displayName: 'Setup & testing: Full build matrix'
-      condition: |
-        or(contains(variables['Build.SourceBranch'], 'refs/heads/release'),
-        eq(variables['Build.SourceBranch'], 'refs/heads/master'),
-        eq(variables['Build.Reason'], 'Schedule'))
+      displayName: '(matrix)'
+      condition: eq(variables.master_or_release, 'True')
 
       pool:
         vmImage: 'ubuntu-latest'
       strategy:
         matrix:
           minimum_dependencies:
-            FACET_V_PYTHON_BUILD: '3.6'
-            FACET_V_PANDAS: '==0.24'
-            FACET_V_SKLEARN: '==0.21.*'
-            FACET_V_JOBLIB: '==0.13'
-            FACET_V_NUMPY: '==1.16'
-            FACET_V_SHAP: '==0.34'
+            FACET_V_PYTHON_BUILD: '=3.6.*'
+            FACET_V_PYTHON: '=3.6.*'
+            FACET_V_PANDAS: '=0.24.*'
+            FACET_V_MATPLOT: '=3.0.*'
+            FACET_V_NUMPY: '=1.16.*'
+            FACET_V_SCIPY: '=1.2.*'
+            FACET_V_SKLEARN: '=0.21.*'
+            FACET_V_BORUTA: '=0.3.*'
+            FACET_V_LGBM: '=2.2.*'
+            FACET_V_JOBLIB: '=0.13.*'
+            FACET_V_SHAP: '=0.34'
           maximum_dependencies:
-            FACET_V_PYTHON_BUILD: '3.8'
-            FACET_V_SKLEARN: '==0.23'
-            FACET_V_PANDAS: '==1.0.0'
-            FACET_V_NUMPY: '=>1.16'
-            FACET_V_SHAP: '==0.35'
+            FACET_V_PYTHON_BUILD: '=3.8.*'
+            FACET_V_PYTHON: '=3.8.5'
+            FACET_V_PANDAS: '=1.1.*'
+            FACET_V_NUMPY: '=1.19.*'
+            FACET_V_SCIPY: '=1.5.*'
+            FACET_V_MATPLOT: '=3.3.*'
+            FACET_V_SKLEARN: '>=0.23.1,<0.24'
+            FACET_V_BORUTA: '=0.3.*'
+            FACET_V_LGBM: '=3.0.*'
+            FACET_V_JOBLIB: '=0.16.*'
+            FACET_V_SHAP: '=0.35'
           unconstrained_dependencies:
-            FACET_V_PYTHON_BUILD: '>=3.6'
-            FACET_V_PANDAS: '=>0.24'
-            FACET_V_SKLEARN: '=>0.21'
-            FACET_V_JOBLIB: '=>0.13'
-            FACET_V_NUMPY: '=>1.16'
-            FACET_V_SHAP: '=>0.34'
+            FACET_V_PYTHON_BUILD: '>=3.6,<4'
+            FACET_V_PYTHON: '>=3.6'
+            FACET_V_PANDAS: '>=0.24'
+            FACET_V_NUMPY: '>=1.16'
+            FACET_V_SCIPY: '>=1.2'
+            FACET_V_MATPLOT: '>=3'
+            FACET_V_SKLEARN: '>=0.23.1'
+            FACET_V_BORUTA: '>=0.3'
+            FACET_V_LGBM: '>=2.2'
+            FACET_V_JOBLIB: '>=0.13'
+            FACET_V_SHAP: '=0.35'
 
       steps:
         - task: UsePythonVersion@0
@@ -175,26 +252,29 @@ stages:
         - checkout: self
         - checkout: sklearndf
         - checkout: pytools
+
         - script: dir $(Build.SourcesDirectory)
 
         - task: Bash@3
           inputs:
             targetType: 'inline'
             script: |
               eval "$(conda shell.bash hook)"
+
               export PYTHONPATH=$(System.DefaultWorkingDirectory)/facet/src/
               export FACET_V_PYTHON_BUILD=$(FACET_V_PYTHON_BUILD)
               export FACET_V_PANDAS=$(FACET_V_PANDAS)
-              export FACET_V_SKLEARN=$(FACET_V_SKLEARN)
               export FACET_V_JOBLIB=$(FACET_V_JOBLIB)
               export FACET_V_NUMPY=$(FACET_V_NUMPY)
-              export FACET_V_SHAP=$(FACET_V_SHAP)
-              conda activate facet-develop
+
               conda install -y conda-build
+
               cd $(System.DefaultWorkingDirectory)/pytools/
               make package
+
               cd $(System.DefaultWorkingDirectory)/sklearndf/
               make package
+
               cd $(System.DefaultWorkingDirectory)/facet/
               make package
           displayName: "Build conda package"

conda-build/meta.yaml

@@ -3,25 +3,26 @@ package:
   version: 1.0.0
 
 source:
-  git_url: {{FACET_PATH}}/facet #todo: change to final
+  git_url: {{FACET_PATH}}/facet
 
 build:
   noarch: python
   script: "python -m pip install . --no-deps --ignore-installed -vv "
 
 requirements:
   host:
-    - pip
-    - python={{ environ.get('FACET_V_PYTHON_BUILD', '3.7') }}
+    - pip>=19.1
+    - python{{ environ.get('FACET_V_PYTHON_BUILD', '=3.8.*') }}
   run:
-    - python>=3.6,<3.8
-    - pandas{{ environ.get('FACET_V_PANDAS', '>=0.24') }}
-    - numpy{{ environ.get('FACET_V_NUMPY', '>=1.16') }}
-    - matplotlib{{ environ.get('FACET_V_MATPLOT', '>=3') }}
-    - shap{{ environ.get('FACET_V_SHAP', '>=0.34') }}
-    - scikit-learn{{ environ.get('FACET_V_SKLEARN', '>=0.21,<=0.22') }}
-    - gamma-pytools=1.0
-    - gamma-sklearndf=1.0
+    - python{{ environ.get('FACET_V_PYTHON', '>=3.6,<4') }}
+    - pandas{{ environ.get('FACET_V_PANDAS', '>=0.24,<1.2') }}
+    - numpy{{ environ.get('FACET_V_NUMPY', '>=1.16,<1.20') }}
+    - scipy{{ environ.get('FACET_V_SCIPY', '>=1.2,<1.6') }}
+    - matplotlib{{ environ.get('FACET_V_MATPLOT', '>=3.1,<3.4') }}
+    - scikit-learn{{ environ.get('FACET_V_SKLEARN', '>=0.21,<0.24') }}
+    - shap{{ environ.get('FACET_V_SHAP', '>=0.34,<0.36') }}
+    - gamma-pytools=1.0.*
+    - gamma-sklearndf=1.0.*
     - pyyaml>=5
 test:
   imports:
@@ -32,8 +33,12 @@ test:
     - facet.validation
     - facet.simulation
   requires:
-    - pytest=5.2
+    - pytest=5.2.*
   commands:
+    - conda list
     - python -c 'import facet;
-      import os;
-      assert facet.__version__ == os.environ["PKG_VERSION"]'
+                 import os;
+                 assert facet.__version__ == os.environ["PKG_VERSION"]'
+    - pytest -vs ${FACET_PATH}/pytools/test
+    - pytest -vs ${FACET_PATH}/sklearndf/test
+    - pytest -vs ${FACET_PATH}/facet/test

environment.yml

@@ -6,36 +6,35 @@ dependencies:
   - conda-build
   - conda-verify
   - black >= 19.3
-  - boruta_py = 0.3.*
+  - boruta_py >= 0.3.*
   - docutils
-  - joblib = 0.13.*
+  - joblib >= 0.13.*
   - jupyter >= 1.0
-  - lightgbm = 2.3.*
-  - matplotlib = 3.1.*
-  - m2r = 0.2.*
-  - nbsphinx=0.4.*
-  - numpy = 1.16.*
-  - pandas = 0.24.*
+  - lightgbm >= 2.3.*
+  - matplotlib >= 3.1.*
+  - m2r >= 0.2.*
+  - nbsphinx >= 0.4.*
+  - numpy >= 1.16.*
+  - pandas >= 0.24.*
   - pip >= 19.1
-  - pluggy = 0.13.*
-  - pytest = 5.2.0
-  - pytest-cov = 2.8.*
-  - python = 3.7.*
-  - pyyaml = 5.1.*
-  - rsa = 4.0.*
+  - pluggy >= 0.13.*
+  - pytest >= 5.2.0
+  - pytest-cov >= 2.8.*
+  - python >=3.8,<4.0
+  - pyyaml >= 5.1.*
   - scikit-learn = 0.23.*
-  - scipy = 1.2.*
-  - seaborn = 0.10.*
-  - shap = 0.34.*
-  - sphinx = 3.1.*
-  - sphinx-autodoc-typehints = 1.11.*
-  - sphinx-automodapi = 0.12.*
-  - sphinx_rtd_theme = 0.5.*
-  - tableone=0.7.*
-  - xlrd = 1.2.*
-  - yaml = 0.1.*
+  - scipy >= 1.2.*
+  - seaborn >= 0.10.*
+  - shap >= 0.34.*
+  - sphinx = 3.*
+  - sphinx-autodoc-typehints >= 1.11.*
+  - sphinx-automodapi >= 0.12.*
+  - sphinx_rtd_theme >= 0.5.*
+  - tableone >= 0.7.*
+  - typing_inspect >= 0.4
+  - xlrd >= 1.2.*
+  - yaml >= 0.1.*
   - pip:
       - pre_commit == 1.16.*
-      - punch.py==1.6.*
+      - punch.py == 1.6.*
       - pydata-sphinx-theme == 0.3.*
-      - typing-inspect == 0.4.*