[Draft] Python Wheels for PyPi

.github/workflows/build-and-test.yml

@@ -6,6 +6,9 @@ on:
  pull_request:
  branches: [ master ]
 
+env:
+ CMDSTAN_VERSION: "2.26.1"
+
 jobs:
  build-and-test-python:
 
@@ -21,16 +24,34 @@ jobs:
  uses: actions/setup-python@v2
  with:
  python-version: ${{ matrix.python-version }}
+ - name: "Restore pip cache"
+ id: cache-pip
+ uses: actions/cache@v2
+ with:
+ path: $HOME/.cache/pip
+ key: ${{ runner.os }}-pip-${{ hashFiles('**/python/requirements.txt') }}-v1
+ restore-keys: |
+ ${{ runner.os }}-pip-
+ - name: "Restore cmdstan cache"
+ id: cache-cmdstan
+ uses: actions/cache@v2
+ with:
+ path: $HOME/.cmdstan
+ key: ${{ runner.os }}-cmdstan-${{ env.CMDSTAN_VERSION }}-v1
+ - name: "Download cmdstan"
+ if: steps.cache-cmdstan.outputs.cache-hit != 'true'
+ run: |
+ wget https://github.com/stan-dev/cmdstan/releases/download/v${{ env.CMDSTAN_VERSION }}/cmdstan-${{ env.CMDSTAN_VERSION }}.tar.gz -O /tmp/cmdstan.tar.gz &> /dev/null
+ mkdir $HOME/.cmdstan
+ tar -xf /tmp/cmdstan.tar.gz -C $HOME/.cmdstan &> /dev/null
  - name: Install and test
  run: |
  pip install -U -r python/requirements.txt dask[dataframe] distributed
- cd python && python setup.py develop test
+ cd python
+ STAN_BACKEND=PYSTAN python setup.py develop test
  python setup.py clean
  rm -rf prophet/stan_model
- wget https://github.com/stan-dev/cmdstan/releases/download/v2.26.1/cmdstan-2.26.1.tar.gz -O /tmp/cmdstan.tar.gz > /dev/null
- tar -xvf /tmp/cmdstan.tar.gz -C /tmp > /dev/null
- make -C /tmp/cmdstan-2.26.1/ build > /dev/null
- CMDSTAN=/tmp/cmdstan-2.26.1 STAN_BACKEND=CMDSTANPY python setup.py develop test
+ STAN_BACKEND=CMDSTANPY python setup.py develop test
 
  build-and-test-r:
 

.github/workflows/wheel.yml

@@ -0,0 +1,106 @@
+name: "Create Python Wheels"
+
+on:
+ release:
+ types: [ created ]
+ workflow_dispatch: {}
+
+env:
+ STAN_BACKEND: "PYSTAN,CMDSTANPY"
+ CMDSTAN_VERSION: "2.26.1"
+
+jobs:
+ make-wheels-macos-linux:
+ name: ${{ matrix.python-version }}-${{ matrix.architecture }}-${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os:
+ - "macos-latest"
+ - "ubuntu-latest"
+ python-version:
+ - "3.6"
+ - "3.7"
+ - "3.8"
+ architecture:
+ - x64
+
+ fail-fast: false
+
+ steps:
+ - name: "Get OS version (Linux)"
+ if: startsWith(runner.os, 'Linux')
+ run: |
+ echo "OS_VERSION=`lsb_release -sr`" >> $GITHUB_ENV
+ echo "PIP_DEFAULT_CACHE=$HOME/.cache/pip" >> $GITHUB_ENV
+ echo "DEFAULT_HOME=$HOME" >> $GITHUB_ENV
+
+ - name: "Get OS version (macOS)"
+ if: startsWith(runner.os, 'macOS')
+ run: |
+ echo "OS_VERSION=`sw_vers -productVersion`" >> $GITHUB_ENV
+ echo "PIP_DEFAULT_CACHE=$HOME/Library/Caches/pip" >> $GITHUB_ENV
+ echo "DEFAULT_HOME=$HOME" >> $GITHUB_ENV
+
+ - name: "Checkout repo"
+ uses: actions/checkout@v2
+
+ - name: "Set up Python"
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ architecture: ${{ matrix.architecture }}
+
+ - name: "Restore pip cache"
+ id: cache-pip
+ uses: actions/cache@v2
+ with:
+ path: ${{ env.PIP_DEFAULT_CACHE }}
+ key: ${{ runner.os }}-pip-${{ hashFiles('**/python/requirements.txt') }}-v1
+ restore-keys: |
+ ${{ runner.os }}-pip-
+
+ - name: "Install pip"
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install cibuildwheel build
+
+ - name: "Restore cmdstan cache"
+ id: cache-cmdstan
+ uses: actions/cache@v2
+ with:
+ path: ${{ env.DEFAULT_HOME }}/.cmdstan
+ key: ${{ runner.os }}-cmdstan-${{ env.CMDSTAN_VERSION }}-v1
+
+ - name: "Download cmdstan"
+ if: steps.cache-cmdstan.outputs.cache-hit != 'true'
+ run: |
+ wget https://github.com/stan-dev/cmdstan/releases/download/v${{ env.CMDSTAN_VERSION }}/cmdstan-${{ env.CMDSTAN_VERSION }}.tar.gz -O /tmp/cmdstan.tar.gz &> /dev/null
+ mkdir $HOME/.cmdstan
+ tar -xf /tmp/cmdstan.tar.gz -C $HOME/.cmdstan &> /dev/null
+
+ - name: "Build wheel"
+ run: |
+ cd python && python -m cibuildwheel --output-dir wheelhouse
+ env:
+ CIBW_ENVIRONMENT: >
+ STAN_BACKEND="${{ env.STAN_BACKEND }}"
+ CMDSTAN_VERSION=${{ env.CMDSTAN_VERSION }}
+ # Linux builds run in a Docker container, need to point the cache to the host machine.
+ CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+ CIBW_ENVIRONMENT_LINUX: >
+ STAN_BACKEND="${{ env.STAN_BACKEND }}"
+ CMDSTAN_VERSION=${{ env.CMDSTAN_VERSION }}
+ HOME="/host/${{ env.DEFAULT_HOME }}"
+ PIP_CACHE_DIR="/host/${{ env.PIP_DEFAULT_CACHE }}"
+ CIBW_BEFORE_ALL_LINUX: sudo chmod -R a+rwx ${{ env.PIP_DEFAULT_CACHE }}
+ CIBW_ARCHS: native
+ CIBW_BUILD_FRONTEND: build
+ CIBW_TEST_REQUIRES: pytest
+ CIBW_TEST_COMMAND: pytest --pyargs prophet
+
+ - name: "Upload wheel as artifact"
+ uses: actions/upload-artifact@v2
+ with:
+ name: ${{ matrix.os }}-wheel
+ path: "./**/*.whl"

python/prophet/models.py

@@ -12,7 +12,6 @@
 from pathlib import Path
 import pickle
 import pkg_resources
-import os
 
 import logging
 logger = logging.getLogger('prophet.models')
@@ -53,30 +52,20 @@ def fit(self, stan_init, stan_data, **kwargs) -> dict:
  def sampling(self, stan_init, stan_data, samples, **kwargs) -> dict:
  pass
 
- @staticmethod
- @abstractmethod
- def build_model(target_dir, model_dir):
- pass
-
 
 class CmdStanPyBackend(IStanBackend):
+ CMDSTAN_VERSION = "2.26.1"
+ def __init__(self):
+ super().__init__()
+ import cmdstanpy
+ cmdstanpy.set_cmdstan_path(
+ pkg_resources.resource_filename("prophet", f"stan_model/cmdstan-{self.CMDSTAN_VERSION}")
+ )
 
  @staticmethod
  def get_type():
  return StanBackendEnum.CMDSTANPY.name
 
- @staticmethod
- def build_model(target_dir, model_dir):
- from shutil import copy
- import cmdstanpy
- model_name = 'prophet.stan'
- target_name = 'prophet_model.bin'
-
- sm = cmdstanpy.CmdStanModel(
- stan_file=os.path.join(model_dir, model_name))
- sm.compile()
- copy(sm.exe_file, os.path.join(target_dir, target_name))
-
  def load_model(self):
  import cmdstanpy
  model_file = pkg_resources.resource_filename(
@@ -87,7 +76,7 @@ def load_model(self):
 
  def fit(self, stan_init, stan_data, **kwargs):
  (stan_init, stan_data) = self.prepare_data(stan_init, stan_data)
- 
+
  if 'inits' not in kwargs and 'init' in kwargs:
  kwargs['inits'] = self.prepare_data(kwargs['init'], stan_data)[0]
 
@@ -120,14 +109,13 @@ def fit(self, stan_init, stan_data, **kwargs):
 
  def sampling(self, stan_init, stan_data, samples, **kwargs) -> dict:
  (stan_init, stan_data) = self.prepare_data(stan_init, stan_data)
- 
+
  if 'inits' not in kwargs and 'init' in kwargs:
  kwargs['inits'] = self.prepare_data(kwargs['init'], stan_data)[0]
 
  args = dict(
  data=stan_data,
  inits=stan_init,
- algorithm='Newton' if stan_data['T'] < 100 else 'LBFGS',
  )
 
  if 'chains' not in kwargs:
@@ -136,7 +124,7 @@ def sampling(self, stan_init, stan_data, samples, **kwargs) -> dict:
  kwargs['iter_sampling'] = iter_half
  if 'iter_warmup' not in kwargs:
  kwargs['iter_warmup'] = iter_half
- 
+
  args.update(kwargs)
 
  self.stan_fit = self.model.sample(**args)
@@ -181,7 +169,7 @@ def prepare_data(init, data) -> Tuple[dict, dict]:
  'sigma_obs': init['sigma_obs']
  }
  return (cmdstanpy_init, cmdstanpy_data)
- 
+
  @staticmethod
  def stan_to_dict_numpy(column_names: Tuple[str, ...], data: 'np.array'):
  import numpy as np
@@ -235,17 +223,6 @@ class PyStanBackend(IStanBackend):
  def get_type():
  return StanBackendEnum.PYSTAN.name
 
- @staticmethod
- def build_model(target_dir, model_dir):
- import pystan
- model_name = 'prophet.stan'
- target_name = 'prophet_model.pkl'
- with open(os.path.join(model_dir, model_name)) as f:
- model_code = f.read()
- sm = pystan.StanModel(model_code=model_code)
- with open(os.path.join(target_dir, target_name), 'wb') as f:
- pickle.dump(sm, f, protocol=pickle.HIGHEST_PROTOCOL)
-
  def sampling(self, stan_init, stan_data, samples, **kwargs) -> dict:
 
  args = dict(

python/pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+requires = [
+ "setuptools>=42",
+ "wheel",
+ "pystan~=2.19.1.1",
+ "cmdstanpy==0.9.77",
+]
+build-backend = "setuptools.build_meta"

python/requirements.txt

@@ -1,12 +1,14 @@
 Cython>=0.22
-cmdstanpy==0.9.68
+cmdstanpy==0.9.77
 pystan~=2.19.1.1
 numpy>=1.15.4
 pandas>=1.0.4
 matplotlib>=2.0.0
 LunarCalendar>=0.0.9
 convertdate>=2.1.2
 holidays>=0.11.3.1
+setuptools>=42
 setuptools-git>=1.2
 python-dateutil>=2.8.0
 tqdm>=4.36.1
+wheel>=0.37.0