Start uncythonization

.github/workflows/conda.yml

@@ -4,10 +4,6 @@ on:
     branches:
       - main
   pull_request:
-    paths:
-      - setup.py
-      - continuous_integration/recipes/**
-      - .github/workflows/conda.yml
 
 # When this workflow is queued, automatically cancel any previous running
 # or pending jobs from the same branch
@@ -56,13 +52,13 @@ jobs:
           conda mambabuild continuous_integration/recipes/distributed \
                            --channel dask/label/dev \
                            --no-anaconda-upload \
-                           --output-folder .
+                           --output-folder build
 
           # dask pre-release build
           conda mambabuild continuous_integration/recipes/dask \
                            --channel dask/label/dev \
                            --no-anaconda-upload \
-                           --output-folder .
+                           --output-folder build
       - name: Upload conda packages
         if: |
           github.event_name == 'push'
@@ -71,8 +67,20 @@ jobs:
         env:
           ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }}
         run: |
+          # convert distributed to other architectures
+          cd build && conda convert linux-64/*.tar.bz2 -p osx-64 \
+                                                       -p osx-arm64 \
+                                                       -p linux-ppc64le \
+                                                       -p linux-aarch64 \
+                                                       -p win-64
+
           # install anaconda for upload
           mamba install anaconda-client
 
           anaconda upload --label dev noarch/*.tar.bz2
           anaconda upload --label dev linux-64/*.tar.bz2
+          anaconda upload --label dev linux-aarch64/*.tar.bz2
+          anaconda upload --label dev linux-ppc64le/*.tar.bz2
+          anaconda upload --label dev osx-64/*.tar.bz2
+          anaconda upload --label dev osx-arm64/*.tar.bz2
+          anaconda upload --label dev win-64/*.tar.bz2

continuous_integration/recipes/dask/meta.yaml

@@ -23,7 +23,7 @@ requirements:
   run:
     - python >=3.8
     - dask-core >={{ dask_version }}
-    - distributed {{ version }}=*{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}*
+    - distributed {{ version }}=*_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
     - cytoolz >=0.8.2
     - numpy >=1.18
     - pandas >=1.0

continuous_integration/recipes/distributed/conda_build_config.yaml

@@ -1,6 +1,3 @@
 python:
     - 3.8
     - 3.9
-cython_enabled:
-    - True   # [linux]
-    - False

continuous_integration/recipes/distributed/meta.yaml

@@ -2,19 +2,23 @@
 {% set new_patch = major_minor_patch[2] | int + 1 %}
 {% set version = (major_minor_patch[:2] + [new_patch]) | join('.') + environ.get('VERSION_SUFFIX', '') %}
 {% set dask_version = environ.get('DASK_CORE_VERSION', '0.0.0.dev') %}
-{% set build_ext = "cython" %}  # [cython_enabled]
-{% set build_ext = "python" %}  # [not cython_enabled]
 
 
 package:
-  name: distributed-split
+  name: distributed
   version: {{ version }}
 
 source:
   git_url: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}
+  string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+  script: {{ PYTHON }} -m pip install . -vv
+  entry_points:
+    - dask-scheduler = distributed.cli.dask_scheduler:go
+    - dask-ssh = distributed.cli.dask_ssh:go
+    - dask-worker = distributed.cli.dask_worker:go
 
 outputs:
   - name: distributed-impl
@@ -37,27 +41,20 @@ outputs:
     version: {{ version }}
     build:
       number: {{ GIT_DESCRIBE_NUMBER }}
-      string: py_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}_{{ build_ext }}  # [not cython_enabled]
-      string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}_{{ build_ext }}  # [cython_enabled]
-      noarch: python                              # [not cython_enabled]
-      skip: True                                  # [cython_enabled and py<38]
+      string: py_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}_{{ build_ext }}
+      noarch: python
       script: >
         python -m pip install . -vv --no-deps
         --install-option="--with-cython=profile"  # [cython_enabled]
-      track_features:             # [cython_enabled]
-        - cythonized-scheduler    # [cython_enabled]
       entry_points:
         - dask-scheduler = distributed.cli.dask_scheduler:main
         - dask-ssh = distributed.cli.dask_ssh:main
         - dask-worker = distributed.cli.dask_worker:main
     requirements:
-      build:
-        - {{ compiler('c') }}                     # [cython_enabled]
       host:
         - python >=3.8
         - pip
         - setuptools
-        - cython                                  # [cython_enabled]
       run:
         - python >=3.8
         - click >=6.6

distributed/scheduler.py

@@ -1388,9 +1388,7 @@ class SchedulerState:
 
     All events are handled quickly, in linear time with respect to their
     input (which is often of constant size) and generally within a
-    millisecond. Additionally when Cythonized, this can be faster still.
-    To accomplish this the scheduler tracks a lot of state.  Every
-    operation maintains the consistency of this state.
+    millisecond.
 
     Users typically do not interact with ``Transitions`` directly. Instead
     users interact with the ``Client``, which in turn engages the
@@ -1480,7 +1478,6 @@ def __init__(
         self.extensions = {}
         self.host_info = host_info
         self.idle = SortedDict()
-        # Note: cython.cast, not typing.cast!
         self.n_tasks = 0
         self.resources = resources
         self.saturated = set()