[Draft] Python Wheels for PyPi

.github/workflows/wheel.yml

@@ -0,0 +1,61 @@
+name: "Create Python Wheels"
+
+on:
+ push:
+ branches: [ master ]
+ pull_request:
+ branches: [ master ]
+ # release:
+ # types: [created]
+
+env:
+ STAN_BACKEND: "PYSTAN,CMDSTANPY"
+ # Env vars, required because CIBW may build inside a Docker container
+ CIBW_ENVIRONMENT: STAN_BACKEND="PYSTAN,CMDSTANPY"
+ # Python versions for cibuildwheel
+ CIBW_BUILD: "cp38-*"
+ # Architectures for cibuildwheel
+ CIBW_ARCHS: "native"
+ # Use pypa/build as the build frontend
+ CIBW_BUILD_FRONTEND: "build"
+
+jobs:
+ make-wheels-macos-linux:
+ name: ${{ matrix.python-version }}-${{ matrix.architecture }}-${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os:
+ - "macos-latest"
+ - "ubuntu-latest"
+ python-version:
+ - "3.8"
+ architecture:
+ - x64
+
+ fail-fast: false
+
+ steps:
+ - name: "Checkout repo"
+ uses: actions/checkout@v2
+
+ - name: "Set up Python"
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ architecture: ${{ matrix.architecture }}
+
+ - name: "Install cibuildwheel"
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install cibuildwheel build
+
+ - name: "Create wheel"
+ run: |
+ cd python && python -m cibuildwheel --output-dir wheelhouse
+
+ - name: "Upload wheel as artifact"
+ uses: actions/upload-artifact@v2
+ with:
+ name: ${{ matrix.os }}-wheel
+ path: "./**/*.whl"

python/prophet/models.py

@@ -60,6 +60,13 @@ def build_model(target_dir, model_dir):
 
 
 class CmdStanPyBackend(IStanBackend):
+ CMDSTAN_VERSION = "2.26.1"
+ def __init__(self):
+ super().__init__()
+ import cmdstanpy
+ cmdstanpy.set_cmdstan_path(
+ pkg_resources.resource_filename("prophet", f"stan_model/cmdstan-{self.CMDSTAN_VERSION}")
+ )
 
  @staticmethod
  def get_type():
@@ -87,7 +94,7 @@ def load_model(self):
 
  def fit(self, stan_init, stan_data, **kwargs):
  (stan_init, stan_data) = self.prepare_data(stan_init, stan_data)
- 
+
  if 'inits' not in kwargs and 'init' in kwargs:
  kwargs['inits'] = self.prepare_data(kwargs['init'], stan_data)[0]
 
@@ -120,14 +127,13 @@ def fit(self, stan_init, stan_data, **kwargs):
 
  def sampling(self, stan_init, stan_data, samples, **kwargs) -> dict:
  (stan_init, stan_data) = self.prepare_data(stan_init, stan_data)
- 
+
  if 'inits' not in kwargs and 'init' in kwargs:
  kwargs['inits'] = self.prepare_data(kwargs['init'], stan_data)[0]
 
  args = dict(
  data=stan_data,
  inits=stan_init,
- algorithm='Newton' if stan_data['T'] < 100 else 'LBFGS',
  )
 
  if 'chains' not in kwargs:
@@ -136,7 +142,7 @@ def sampling(self, stan_init, stan_data, samples, **kwargs) -> dict:
  kwargs['iter_sampling'] = iter_half
  if 'iter_warmup' not in kwargs:
  kwargs['iter_warmup'] = iter_half
- 
+
  args.update(kwargs)
 
  self.stan_fit = self.model.sample(**args)
@@ -181,7 +187,7 @@ def prepare_data(init, data) -> Tuple[dict, dict]:
  'sigma_obs': init['sigma_obs']
  }
  return (cmdstanpy_init, cmdstanpy_data)
- 
+
  @staticmethod
  def stan_to_dict_numpy(column_names: Tuple[str, ...], data: 'np.array'):
  import numpy as np

python/pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+requires = [
+ "setuptools>=42",
+ "wheel",
+ "pystan~=2.19.1.1",
+ "cmdstanpy==0.9.77"
+]
+build-backend = "setuptools.build_meta"

python/requirements.txt

@@ -1,12 +1,14 @@
 Cython>=0.22
-cmdstanpy==0.9.68
+cmdstanpy==0.9.77
 pystan~=2.19.1.1
 numpy>=1.15.4
 pandas>=1.0.4
 matplotlib>=2.0.0
 LunarCalendar>=0.0.9
 convertdate>=2.1.2
 holidays>=0.10.2
+setuptools>=42
 setuptools-git>=1.2
 python-dateutil>=2.8.0
 tqdm>=4.36.1
+wheel>=0.37.0

python/setup.py

@@ -3,10 +3,11 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-import os.path
+import os
+import pickle
 import platform
 import sys
-import os
+from shutil import copy, rmtree
 from pkg_resources import (
  normalize_path,
  working_set,
@@ -16,26 +17,77 @@
 from setuptools import setup, find_packages
 from setuptools.command.build_py import build_py
 from setuptools.command.develop import develop
+from setuptools.command.install import install
 from setuptools.command.test import test as test_command
 from typing import List
+from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
 
 PLATFORM = 'unix'
 if platform.platform().startswith('Win'):
  PLATFORM = 'win'
 
 MODEL_DIR = os.path.join('stan', PLATFORM)
 MODEL_TARGET_DIR = os.path.join('prophet', 'stan_model')
-
+CMDSTAN_VERSION = "2.26.1"
 
 def get_backends_from_env() -> List[str]:
- from prophet.models import StanBackendEnum
- return os.environ.get("STAN_BACKEND", StanBackendEnum.PYSTAN.name).split(",")
-
+ return os.environ.get("STAN_BACKEND", "PYSTAN").split(",")
+
+def prune_cmdstan_files(cmdstan_dir):
+ """
+ Remove unnecessary folders from the unbundled cmdstan installation.
+ We only need to keep the files that will be used to execute the model binary at runtime.
+ """
+ remove_dirs = ["stan"]
+ for dirname in remove_dirs:
+ rmtree(os.path.join(cmdstan_dir, dirname))
+ raw_binaries = ["linux-stanc", "mac-stanc", "windows-stanc"] # These are converted into "stanc" in the Make process
+ for fname in raw_binaries:
+ os.remove(os.path.join(cmdstan_dir, f"bin/{fname}"))
+
+def build_cmdstan_model(target_dir):
+ import cmdstanpy
+ cmdstanpy.install_cmdstan(version=CMDSTAN_VERSION, dir=target_dir, overwrite=True)
+ cmdstan_dir = os.path.join(target_dir, f"cmdstan-{CMDSTAN_VERSION}")
+ cmdstanpy.set_cmdstan_path(cmdstan_dir)
+ model_name = 'prophet.stan'
+ target_name = 'prophet_model.bin'
+ sm = cmdstanpy.CmdStanModel(stan_file=os.path.join(MODEL_DIR, model_name))
+ sm.compile()
+ copy(sm.exe_file, os.path.join(target_dir, target_name))
+ prune_cmdstan_files(cmdstan_dir)
+
+def build_pystan_model(target_dir):
+ import pystan
+ model_name = 'prophet.stan'
+ target_name = 'prophet_model.pkl'
+ with open(os.path.join(MODEL_DIR, model_name)) as f:
+ model_code = f.read()
+ sm = pystan.StanModel(model_code=model_code)
+ with open(os.path.join(target_dir, target_name), 'wb') as f:
+ pickle.dump(sm, f, protocol=pickle.HIGHEST_PROTOCOL)
 
 def build_models(target_dir):
- from prophet.models import StanBackendEnum
  for backend in get_backends_from_env():
- StanBackendEnum.get_backend_class(backend).build_model(target_dir, MODEL_DIR)
+ print(f"Compiling {backend} model")
+ if backend == "CMDSTANPY":
+ build_cmdstan_model(target_dir)
+ elif backend == "PYSTAN":
+ build_pystan_model(target_dir)
+
+
+class BDistWheelNonPure(_bdist_wheel):
+ def finalize_options(self):
+ _bdist_wheel.finalize_options(self)
+ self.root_is_pure = False
+
+
+class InstallPlatlib(install):
+ """Ensure wheel has correct platlib specification when being repaired by auditwheel."""
+ def finalize_options(self):
+ install.finalize_options(self)
+ if self.distribution.has_ext_modules():
+ self.install_lib = self.install_platlib
 
 
 class BuildPyCommand(build_py):
@@ -129,15 +181,15 @@ def with_project_on_sys_path(self, func):
  author_email='sjtz@pm.me',
  license='MIT',
  packages=find_packages(),
- setup_requires=[
- ],
  install_requires=install_requires,
  python_requires='>=3',
  zip_safe=False,
  include_package_data=True,
  cmdclass={
+ 'bdist_wheel': BDistWheelNonPure,
  'build_py': BuildPyCommand,
  'develop': DevelopCommand,
+ 'install': InstallPlatlib,
  'test': TestCommand,
  },
  test_suite='prophet.tests',