(Stage 1) Python Wheels for PyPi (#2010)

.github/workflows/build-and-test.yml

@@ -6,6 +6,9 @@ on:
   pull_request:
     branches: [ master ]
 
+env:
+  CMDSTAN_VERSION: "2.26.1"
+
 jobs:
   build-and-test-python:
 
@@ -21,16 +24,34 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: "Restore pip cache"
+      id: cache-pip
+      uses: actions/cache@v2
+      with:
+        path: $HOME/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/python/requirements.txt') }}-v1
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: "Restore cmdstan cache"
+      id: cache-cmdstan
+      uses: actions/cache@v2
+      with:
+        path: $HOME/.cmdstan
+        key: ${{ runner.os }}-cmdstan-${{ env.CMDSTAN_VERSION }}-v1
+    - name: "Download cmdstan"
+      if: steps.cache-cmdstan.outputs.cache-hit != 'true'
+      run: |
+        wget https://github.com/stan-dev/cmdstan/releases/download/v${{ env.CMDSTAN_VERSION }}/cmdstan-${{ env.CMDSTAN_VERSION }}.tar.gz -O /tmp/cmdstan.tar.gz &> /dev/null
+        mkdir $HOME/.cmdstan
+        tar -xf /tmp/cmdstan.tar.gz -C $HOME/.cmdstan &> /dev/null
     - name: Install and test
       run: |
         pip install -U -r python/requirements.txt dask[dataframe] distributed
-        cd python && python setup.py develop test
+        cd python
+        STAN_BACKEND=PYSTAN python setup.py develop test
         python setup.py clean
         rm -rf prophet/stan_model
-        wget https://github.com/stan-dev/cmdstan/releases/download/v2.26.1/cmdstan-2.26.1.tar.gz -O /tmp/cmdstan.tar.gz > /dev/null
-        tar -xvf /tmp/cmdstan.tar.gz -C /tmp > /dev/null
-        make -C /tmp/cmdstan-2.26.1/ build > /dev/null
-        CMDSTAN=/tmp/cmdstan-2.26.1 STAN_BACKEND=CMDSTANPY python setup.py develop test
+        STAN_BACKEND=CMDSTANPY python setup.py develop test
 
   build-and-test-r:
 

.github/workflows/wheel.yml

@@ -0,0 +1,106 @@
+name: "Create Python Wheels"
+
+on:
+  release:
+    types: [ created ]
+  workflow_dispatch: {}
+
+env:
+  STAN_BACKEND: "PYSTAN,CMDSTANPY"
+  CMDSTAN_VERSION: "2.26.1"
+
+jobs:
+  make-wheels-macos-linux:
+    name: ${{ matrix.python-version }}-${{ matrix.architecture }}-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - "macos-latest"
+          - "ubuntu-latest"
+        python-version:
+          - "3.6"
+          - "3.7"
+          - "3.8"
+        architecture:
+          - x64
+
+      fail-fast: false
+
+    steps:
+      - name: "Get OS version (Linux)"
+        if: startsWith(runner.os, 'Linux')
+        run: |
+          echo "OS_VERSION=`lsb_release -sr`" >> $GITHUB_ENV
+          echo "PIP_DEFAULT_CACHE=$HOME/.cache/pip" >> $GITHUB_ENV
+          echo "DEFAULT_HOME=$HOME" >> $GITHUB_ENV
+
+      - name: "Get OS version (macOS)"
+        if: startsWith(runner.os, 'macOS')
+        run: |
+          echo "OS_VERSION=`sw_vers -productVersion`" >> $GITHUB_ENV
+          echo "PIP_DEFAULT_CACHE=$HOME/Library/Caches/pip" >> $GITHUB_ENV
+          echo "DEFAULT_HOME=$HOME" >> $GITHUB_ENV
+
+      - name: "Checkout repo"
+        uses: actions/checkout@v2
+
+      - name: "Set up Python"
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: ${{ matrix.architecture }}
+
+      - name: "Restore pip cache"
+        id: cache-pip
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.PIP_DEFAULT_CACHE }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/python/requirements.txt') }}-v1
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: "Install pip"
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install cibuildwheel build
+
+      - name: "Restore cmdstan cache"
+        id: cache-cmdstan
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.DEFAULT_HOME }}/.cmdstan
+          key: ${{ runner.os }}-cmdstan-${{ env.CMDSTAN_VERSION }}-v1
+
+      - name: "Download cmdstan"
+        if: steps.cache-cmdstan.outputs.cache-hit != 'true'
+        run: |
+          wget https://github.com/stan-dev/cmdstan/releases/download/v${{ env.CMDSTAN_VERSION }}/cmdstan-${{ env.CMDSTAN_VERSION }}.tar.gz -O /tmp/cmdstan.tar.gz &> /dev/null
+          mkdir $HOME/.cmdstan
+          tar -xf /tmp/cmdstan.tar.gz -C $HOME/.cmdstan &> /dev/null
+
+      - name: "Build wheel"
+        run: |
+          cd python && python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_ENVIRONMENT: >
+            STAN_BACKEND="${{ env.STAN_BACKEND }}"
+            CMDSTAN_VERSION=${{ env.CMDSTAN_VERSION }}
+          # Linux builds run in a Docker container, need to point the cache to the host machine.
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_ENVIRONMENT_LINUX: >
+            STAN_BACKEND="${{ env.STAN_BACKEND }}"
+            CMDSTAN_VERSION=${{ env.CMDSTAN_VERSION }}
+            HOME="/host/${{ env.DEFAULT_HOME }}"
+            PIP_CACHE_DIR="/host/${{ env.PIP_DEFAULT_CACHE }}"
+          CIBW_BEFORE_ALL_LINUX: sudo chmod -R a+rwx ${{ env.PIP_DEFAULT_CACHE }}
+          CIBW_ARCHS: native
+          CIBW_BUILD_FRONTEND: build
+          CIBW_TEST_REQUIRES: pytest
+          CIBW_TEST_COMMAND: pytest --pyargs prophet
+
+      - name: "Upload wheel as artifact"
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.os }}-wheel
+          path: "./**/*.whl"

python/prophet/models.py

@@ -12,7 +12,6 @@
 from pathlib import Path
 import pickle
 import pkg_resources
-import os
 
 import logging
 logger = logging.getLogger('prophet.models')
@@ -53,30 +52,20 @@ def fit(self, stan_init, stan_data, **kwargs) -> dict:
     def sampling(self, stan_init, stan_data, samples, **kwargs) -> dict:
         pass
 
-    @staticmethod
-    @abstractmethod
-    def build_model(target_dir, model_dir):
-        pass
-
 
 class CmdStanPyBackend(IStanBackend):
+    CMDSTAN_VERSION = "2.26.1"
+    def __init__(self):
+        super().__init__()
+        import cmdstanpy
+        cmdstanpy.set_cmdstan_path(
+            pkg_resources.resource_filename("prophet", f"stan_model/cmdstan-{self.CMDSTAN_VERSION}")
+        )
 
     @staticmethod
     def get_type():
         return StanBackendEnum.CMDSTANPY.name
 
-    @staticmethod
-    def build_model(target_dir, model_dir):
-        from shutil import copy
-        import cmdstanpy
-        model_name = 'prophet.stan'
-        target_name = 'prophet_model.bin'
-
-        sm = cmdstanpy.CmdStanModel(
-            stan_file=os.path.join(model_dir, model_name))
-        sm.compile()
-        copy(sm.exe_file, os.path.join(target_dir, target_name))
-
     def load_model(self):
         import cmdstanpy
         model_file = pkg_resources.resource_filename(
@@ -87,7 +76,7 @@ def load_model(self):
 
     def fit(self, stan_init, stan_data, **kwargs):
         (stan_init, stan_data) = self.prepare_data(stan_init, stan_data)
-        
+
         if 'inits' not in kwargs and 'init' in kwargs:
             kwargs['inits'] = self.prepare_data(kwargs['init'], stan_data)[0]
 
@@ -120,14 +109,13 @@ def fit(self, stan_init, stan_data, **kwargs):
 
     def sampling(self, stan_init, stan_data, samples, **kwargs) -> dict:
         (stan_init, stan_data) = self.prepare_data(stan_init, stan_data)
-        
+
         if 'inits' not in kwargs and 'init' in kwargs:
             kwargs['inits'] = self.prepare_data(kwargs['init'], stan_data)[0]
 
         args = dict(
             data=stan_data,
             inits=stan_init,
-            algorithm='Newton' if stan_data['T'] < 100 else 'LBFGS',
         )
 
         if 'chains' not in kwargs:
@@ -136,7 +124,7 @@ def sampling(self, stan_init, stan_data, samples, **kwargs) -> dict:
         kwargs['iter_sampling'] = iter_half
         if 'iter_warmup' not in kwargs:
             kwargs['iter_warmup'] = iter_half
-        
+
         args.update(kwargs)
 
         self.stan_fit = self.model.sample(**args)
@@ -181,7 +169,7 @@ def prepare_data(init, data) -> Tuple[dict, dict]:
             'sigma_obs': init['sigma_obs']
         }
         return (cmdstanpy_init, cmdstanpy_data)
-    
+
     @staticmethod
     def stan_to_dict_numpy(column_names: Tuple[str, ...], data: 'np.array'):
         import numpy as np
@@ -235,17 +223,6 @@ class PyStanBackend(IStanBackend):
     def get_type():
         return StanBackendEnum.PYSTAN.name
 
-    @staticmethod
-    def build_model(target_dir, model_dir):
-        import pystan
-        model_name = 'prophet.stan'
-        target_name = 'prophet_model.pkl'
-        with open(os.path.join(model_dir, model_name)) as f:
-            model_code = f.read()
-        sm = pystan.StanModel(model_code=model_code)
-        with open(os.path.join(target_dir, target_name), 'wb') as f:
-            pickle.dump(sm, f, protocol=pickle.HIGHEST_PROTOCOL)
-
     def sampling(self, stan_init, stan_data, samples, **kwargs) -> dict:
 
         args = dict(

python/pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel",
+    "pystan~=2.19.1.1",
+    "cmdstanpy==0.9.77",
+]
+build-backend = "setuptools.build_meta"

python/requirements.txt

@@ -1,12 +1,14 @@
 Cython>=0.22
-cmdstanpy==0.9.68
+cmdstanpy==0.9.77
 pystan~=2.19.1.1
 numpy>=1.15.4
 pandas>=1.0.4
 matplotlib>=2.0.0
 LunarCalendar>=0.0.9
 convertdate>=2.1.2
 holidays>=0.11.3.1
+setuptools>=42
 setuptools-git>=1.2
 python-dateutil>=2.8.0
 tqdm>=4.36.1
+wheel>=0.37.0