Start building pre-releases with cythonized scheduler (dask#5831)

martindurant · Feb 24, 2022 · 94ebd57 · 94ebd57
1 parent 985bff8
commit 94ebd57
Show file tree

Hide file tree

Showing 4 changed files with 101 additions and 80 deletions.
diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
@@ -4,6 +4,10 @@ on:
     branches:
       - main
   pull_request:
+    paths:
+      - setup.py
+      - continuous_integration/recipes/**
+      - .github/workflows/conda.yml
 
 # When this workflow is queued, automatically cancel any previous running
 # or pending jobs from the same branch
@@ -52,13 +56,13 @@ jobs:
           conda mambabuild continuous_integration/recipes/distributed \
                            --channel dask/label/dev \
                            --no-anaconda-upload \
-                           --output-folder build
+                           --output-folder .
 
           # dask pre-release build
           conda mambabuild continuous_integration/recipes/dask \
                            --channel dask/label/dev \
                            --no-anaconda-upload \
-                           --output-folder build
+                           --output-folder .
       - name: Upload conda packages
         if: |
           github.event_name == 'push'
@@ -67,20 +71,8 @@ jobs:
         env:
           ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }}
         run: |
-          # convert distributed to other architectures
-          cd build && conda convert linux-64/*.tar.bz2 -p osx-64 \
-                                                       -p osx-arm64 \
-                                                       -p linux-ppc64le \
-                                                       -p linux-aarch64 \
-                                                       -p win-64
-
           # install anaconda for upload
           mamba install anaconda-client
 
           anaconda upload --label dev noarch/*.tar.bz2
           anaconda upload --label dev linux-64/*.tar.bz2
-          anaconda upload --label dev linux-aarch64/*.tar.bz2
-          anaconda upload --label dev linux-ppc64le/*.tar.bz2
-          anaconda upload --label dev osx-64/*.tar.bz2
-          anaconda upload --label dev osx-arm64/*.tar.bz2
-          anaconda upload --label dev win-64/*.tar.bz2
diff --git a/continuous_integration/recipes/dask/meta.yaml b/continuous_integration/recipes/dask/meta.yaml
@@ -22,7 +22,7 @@ requirements:
   run:
     - python >=3.8
     - dask-core >={{ dask_version }}
-    - distributed {{ version }}=*_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+    - distributed {{ version }}=*{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}*
     - cytoolz >=0.8.2
     - numpy >=1.18
     - pandas >=1.0

diff --git a/continuous_integration/recipes/distributed/conda_build_config.yaml b/continuous_integration/recipes/distributed/conda_build_config.yaml
@@ -1,3 +1,6 @@
 python:
     - 3.8
     - 3.9
+cython_enabled:
+    - True   # [linux]
+    - False
diff --git a/continuous_integration/recipes/distributed/meta.yaml b/continuous_integration/recipes/distributed/meta.yaml
@@ -2,80 +2,106 @@
 {% set new_patch = major_minor_patch[2] | int + 1 %}
 {% set version = (major_minor_patch[:2] + [new_patch]) | join('.') + environ.get('VERSION_SUFFIX', '') %}
 {% set dask_version = environ.get('DASK_CORE_VERSION', '0.0.0.dev') %}
+{% set build_ext = "cython" %}  # [cython_enabled]
+{% set build_ext = "python" %}  # [not cython_enabled]
 
 
 package:
-  name: distributed
+  name: distributed-split
   version: {{ version }}
 
 source:
   git_url: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}
-  string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
-  script: {{ PYTHON }} -m pip install . -vv
-  entry_points:
-    - dask-scheduler = distributed.cli.dask_scheduler:go
-    - dask-ssh = distributed.cli.dask_ssh:go
-    - dask-worker = distributed.cli.dask_worker:go
 
-requirements:
-  build:
-    - python                                 # [build_platform != target_platform]
-    - cross-python_{{ target_platform }}     # [build_platform != target_platform]
-  host:
-    - python
-    - pip
+outputs:
+  - name: distributed-impl
+    version: {{ version }}
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: {{ build_ext }}
+      noarch: generic
+    test:
+      commands:
+        - exit 0
+    about:
+      home: https://distributed.dask.org
+      summary: A meta-package to select Distributed pre-release build variant
+      license: BSD-3-Clause
+      license_family: BSD
+      license_file: LICENSE.txt
 
-  run:
-    - python
-    - click >=6.6
-    - cloudpickle >=1.5.0
-    - cytoolz >=0.8.2
-    - dask-core >={{ dask_version }}
-    - jinja2
-    - msgpack-python >=0.6.0
-    - packaging >=20.0
-    - psutil >=5.0
-    - pyyaml
-    - sortedcontainers !=2.0.0,!=2.0.1
-    - tblib >=1.6.0
-    - toolz >=0.8.2
-    - tornado >=5  # [py<38]
-    - tornado >=6.0.3  # [py>=38]
-    - zict >=0.1.3
-    - setuptools <60.0.0
-
-  run_constrained:
-    - openssl !=1.1.1e
-
-test:
-  imports:
-    - distributed
-    - distributed.cli
-    - distributed.comm
-    - distributed.deploy
-    - distributed.diagnostics
-    - distributed.protocol
-  commands:
-    - pip check
-    - dask-scheduler --help
-    - dask-ssh --help
-    - dask-worker --help
-  requires:
-    - pip
-
-about:
-  home: https://distributed.dask.org
-  summary: Distributed scheduler for Dask
-  license: BSD-3-Clause
-  license_family: BSD
-  license_file: LICENSE.txt
-  description: |
-    Distributed is a lightweight library for distributed computing in Python.
-    It extends both the concurrent.futures and dask APIs to moderate sized
-    clusters.
-  doc_url: https://distributed.dask.org
-  doc_source_url: https://github.com/dask/distributed/blob/main/docs/source/index.rst
-  dev_url: https://github.com/dask/distributed
+  - name: distributed
+    version: {{ version }}
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: py_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}_{{ build_ext }}  # [not cython_enabled]
+      string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}_{{ build_ext }}  # [cython_enabled]
+      noarch: python                              # [not cython_enabled]
+      skip: True                                  # [cython_enabled and py<38]
+      script: >
+        python -m pip install . -vv --no-deps
+        --install-option="--with-cython=profile"  # [cython_enabled]
+      track_features: {{ "cythonized-scheduler" if cython_enabled else "" }}
+      entry_points:
+        - dask-scheduler = distributed.cli.dask_scheduler:go
+        - dask-ssh = distributed.cli.dask_ssh:go
+        - dask-worker = distributed.cli.dask_worker:go
+    requirements:
+      build:
+        - {{ compiler('c') }}                     # [cython_enabled]
+      host:
+        - python
+        - pip
+        - cython                                  # [cython_enabled]
+      run:
+        - python
+        - click >=6.6
+        - cloudpickle >=1.5.0
+        - cytoolz >=0.8.2
+        - dask-core >={{ dask_version }}
+        - jinja2
+        - msgpack-python >=0.6.0
+        - packaging >=20.0
+        - psutil >=5.0
+        - pyyaml
+        - sortedcontainers !=2.0.0,!=2.0.1
+        - tblib >=1.6.0
+        - toolz >=0.8.2
+        - tornado >=6.0.3
+        - zict >=0.1.3
+        - setuptools <60.0.0
+      run_constrained:
+        - distributed-impl * {{ build_ext }}
+        - openssl !=1.1.1e
+    test:
+      imports:
+        - distributed
+        - distributed.cli
+        - distributed.comm
+        - distributed.deploy
+        - distributed.diagnostics
+        - distributed.protocol
+      commands:
+        - pip check
+        - dask-scheduler --help
+        - dask-ssh --help
+        - dask-worker --help
+        - python -c "from distributed.scheduler import COMPILED; assert COMPILED is {{ cython_enabled }}"
+      requires:
+        - pip
+    about:
+      home: https://distributed.dask.org
+      summary: Distributed scheduler for Dask
+      license: BSD-3-Clause
+      license_family: BSD
+      license_file: LICENSE.txt
+      description: |
+        Distributed is a lightweight library for distributed computing in Python.
+        It extends both the concurrent.futures and dask APIs to moderate sized
+        clusters.
+      doc_url: https://distributed.dask.org
+      doc_source_url: https://github.com/dask/distributed/blob/main/docs/source/index.rst
+      dev_url: https://github.com/dask/distributed