Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup empty tests, add --cores argument #470

Merged
merged 4 commits into from
Oct 8, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 28 additions & 8 deletions cmdstanpy/install_cmdstan.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,9 @@
from time import sleep
from typing import Any, Callable, Dict, Optional

from tqdm.auto import tqdm # type: ignore
from tqdm.auto import tqdm

from cmdstanpy import _DOT_CMDSTAN, _DOT_CMDSTANPY

from cmdstanpy.utils import (
cmdstan_path,
do_command,
Expand Down Expand Up @@ -94,7 +93,7 @@ def clean_all(verbose: bool = False) -> None:
raise CmdStanInstallError(f'Command "make clean-all" failed\n{str(e)}')


def build(verbose: bool = False, progress: bool = True) -> None:
def build(verbose: bool = False, progress: bool = True, cores: int = 1) -> None:
"""
Run command ``make build`` in the current directory, which must be
the home directory of a CmdStan version (or GitHub repo).
Expand All @@ -107,8 +106,10 @@ def build(verbose: bool = False, progress: bool = True) -> None:
Default is ``False``.
:param progress: Boolean value; when ``True`` display progress progress bar.
Default is ``True``.
:param cores: Integer, number of cores to use in the ``make`` command.
Default is 1 core.
"""
cmd = [MAKE, 'build']
cmd = [MAKE, 'build', f'-j{cores}']
try:
if verbose:
do_command(cmd)
Expand Down Expand Up @@ -201,19 +202,26 @@ def compile_example(verbose: bool = False) -> None:
raise CmdStanInstallError(f'Command "make clean-all" failed\n{e}')


def rebuild_cmdstan(verbose: bool = False, progress: bool = True) -> None:
def rebuild_cmdstan(
verbose: bool = False, progress: bool = True, cores: int = 1
) -> None:
"""
Rebuilds the existing CmdStan installation.
This assumes CmdStan has already been installed,
though it need not be installed via CmdStanPy for
this function to work.

:param verbose: Boolean value; when ``True``, show output from make command.
Default is ``False``.
:param progress: Boolean value; when ``True`` display progress progress bar.
Default is ``True``.
:param cores: Integer, number of cores to use in the ``make`` command.
Default is 1 core.
"""
try:
with pushd(cmdstan_path()):
clean_all(verbose)
build(verbose, progress)
build(verbose, progress, cores)
compile_example(verbose)
except ValueError as e:
raise CmdStanInstallError(
Expand All @@ -222,7 +230,11 @@ def rebuild_cmdstan(verbose: bool = False, progress: bool = True) -> None:


def install_version(
cmdstan_version: str, overwrite: bool = False, verbose: bool = False
cmdstan_version: str,
overwrite: bool = False,
verbose: bool = False,
progress: bool = True,
cores: int = 1,
) -> None:
"""
Build specified CmdStan version by spawning subprocesses to
Expand All @@ -245,7 +257,7 @@ def install_version(
)
clean_all(verbose)
print('Rebuilding version {}'.format(cmdstan_version))
build(verbose)
build(verbose, progress=progress, cores=cores)
print('Test model compilation')
compile_example(verbose)
print('Installed {}'.format(cmdstan_version))
Expand Down Expand Up @@ -418,6 +430,12 @@ def parse_cmdline_args() -> Dict[str, Any]:
action='store_true',
help="flag, when specified show progress bar for CmdStan download",
)
parser.add_argument(
"--cores",
default=1,
type=int,
help="number of cores to use while building",
)
if platform.system() == 'Windows':
# use compiler installed with install_cxx_toolchain
# Install a new compiler if compiler not found
Expand Down Expand Up @@ -520,6 +538,8 @@ def main(args: Dict[str, Any]) -> None:
cmdstan_version=cmdstan_version,
overwrite=args['overwrite'],
verbose=args['verbose'],
progress=progress,
cores=args['cores'],
)
except RuntimeError as e:
print(e)
Expand Down
5 changes: 0 additions & 5 deletions cmdstanpy/install_cxx_toolchain.py
Original file line number Diff line number Diff line change
Expand Up @@ -303,11 +303,6 @@ def main(args: Dict[str, Any]) -> None:

if 'progress' in args:
progress = args['progress']
try:
# pylint: disable=unused-import
from tqdm import tqdm # noqa: F401
except (ImportError, ModuleNotFoundError):
progress = False
else:
progress = False

Expand Down
16 changes: 4 additions & 12 deletions cmdstanpy/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,17 @@
from pathlib import Path
from typing import Any, Callable, Dict, List, Mapping, Optional, Union

from tqdm.auto import tqdm # type: ignore

from cmdstanpy import (
_CMDSTAN_REFRESH,
_CMDSTAN_SAMPLING,
_CMDSTAN_WARMUP,
)
from tqdm.auto import tqdm

from cmdstanpy import _CMDSTAN_REFRESH, _CMDSTAN_SAMPLING, _CMDSTAN_WARMUP
from cmdstanpy.cmdstan_args import (
CmdStanArgs,
GenerateQuantitiesArgs,
OptimizeArgs,
SamplerArgs,
VariationalArgs,
)

from cmdstanpy.compiler_opts import CompilerOptions

from cmdstanpy.stanfit import (
CmdStanGQ,
CmdStanMCMC,
Expand All @@ -40,7 +33,6 @@
RunSet,
from_csv,
)

from cmdstanpy.utils import (
EXTENSION,
MaybeDictToFilePath,
Expand Down Expand Up @@ -1353,10 +1345,10 @@ def _run_cmdstan(
) # make the typechecker happy
sampler_args.fixed_param = True

# pylint: disable=no-self-use
@staticmethod
@progbar.wrap_callback
def _wrap_sampler_progress_hook(
self, chain_id: int, total: int
chain_id: int, total: int
) -> Optional[Callable[[str], None]]:
"""Sets up tqdm callback for CmdStan sampler console msgs."""
pbar: tqdm = tqdm(
Expand Down
8 changes: 6 additions & 2 deletions cmdstanpy/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import numpy as np
import pandas as pd
import ujson as json
from tqdm.auto import tqdm # type: ignore
from tqdm.auto import tqdm

from cmdstanpy import (
_CMDSTAN_SAMPLING,
Expand Down Expand Up @@ -990,7 +990,7 @@ def do_command(


def returncode_msg(retcode: int) -> str:
""" interpret retcode"""
"""interpret retcode"""
if retcode < 0:
sig = -1 * retcode
return f'terminated by signal {sig}'
Expand Down Expand Up @@ -1148,6 +1148,7 @@ def install_cmdstan(
compiler: bool = False,
progress: bool = False,
verbose: bool = False,
cores: int = 1,
) -> bool:
"""
Download and install a CmdStan release from GitHub. Downloads the release
Expand Down Expand Up @@ -1176,6 +1177,8 @@ def install_cmdstan(
:param verbose: Boolean value; when ``True``, show console output from all
intallation steps, i.e., download, build, and test CmdStan release.
Default is ``False``.
:param cores: Integer, number of cores to use in the ``make`` command.
Default is 1 core.

:return: Boolean value; ``True`` for success.
"""
Expand All @@ -1187,6 +1190,7 @@ def install_cmdstan(
"compiler": compiler,
"progress": progress,
"dir": dir,
"cores": cores,
}

try:
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ warn_return_any = true

[[tool.mypy.overrides]]
module = [
'tqdm',
'tqdm.auto',
'pandas',
'ujson',
'numpy', # these two are required for py36, which numpy 1.21 doesn't support
Expand Down
1 change: 0 additions & 1 deletion test/test_cmdstan_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -645,7 +645,6 @@ def test_args_fitted_params(self):
class VariationalTest(unittest.TestCase):
def test_args_variational(self):
args = VariationalArgs()
self.assertTrue(True)

args = VariationalArgs(output_samples=1)
args.validate(chains=1)
Expand Down
6 changes: 1 addition & 5 deletions test/test_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from testfixtures import LogCapture, StringComparison

from cmdstanpy.model import CmdStanModel
from cmdstanpy.utils import EXTENSION, cmdstan_path
from cmdstanpy.utils import EXTENSION

HERE = os.path.dirname(os.path.abspath(__file__))
DATAFILES_PATH = os.path.join(HERE, 'data')
Expand Down Expand Up @@ -48,10 +48,6 @@ def do_clean_up(self):
filepath = os.path.join(root, filename)
os.remove(filepath)

def show_cmdstan_version(self):
print('\n\nCmdStan version: {}\n\n'.format(cmdstan_path()))
self.assertTrue(True)

def test_model_good(self):
# compile on instantiation, override model name
model = CmdStanModel(model_name='bern', stan_file=BERN_STAN)
Expand Down
4 changes: 0 additions & 4 deletions test/test_variational.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,9 +198,6 @@ def test_variational_good(self):

self.assertEqual(variational.variational_sample.shape, (1000, 5))

def test_variational_missing_args(self):
self.assertTrue(True)

def test_variational_eta_small(self):
stan = os.path.join(
DATAFILES_PATH, 'variational', 'eta_should_be_small.stan'
Expand All @@ -217,7 +214,6 @@ def test_variational_eta_small(self):
self.assertAlmostEqual(
fabs(variational.variational_params_dict['mu[2]']), 0.09, places=1
)
self.assertTrue(True)

def test_variational_eta_fail(self):
stan = os.path.join(
Expand Down