Skip to content

Commit

Permalink
split up env var setting and activation into separate scripts
Browse files Browse the repository at this point in the history
  • Loading branch information
@msarahan
msarahan committed Nov 13, 2018
1 parent 0dc1025 commit 1ea3af4
Show file tree
Hide file tree
Showing 4 changed files with 206 additions and 161 deletions.
22 changes: 18 additions & 4 deletions conda_build/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,25 +395,39 @@ def update_index(dir_paths, config=None, force=False, check_md5=False, remove=Fa


def debug(recipe_or_package_path_or_metadata_tuples, path=None, test=False, output_id=None, **kwargs):
import os
from fnmatch import fnmatch
from conda_build.conda_interface import string_types
from conda_build.environ import create_env
from conda_build.conda_interface import CONDA_TARBALL_EXTENSIONS, subdir
config = Config()
if isinstance(recipe_or_package_path_or_metadata_tuples, string_types):
metadata_tuples = render(recipe_or_package_path_or_metadata_tuples, **kwargs)
if os.path.splitext(recipe_or_package_path_or_metadata_tuples)[1] in CONDA_TARBALL_EXTENSIONS:
metadata_tuples = render(recipe_or_package_path_or_metadata_tuples, **kwargs)
else:
metadata_tuples = render(recipe_or_package_path_or_metadata_tuples, **kwargs)
else:
metadata_tuples = recipe_or_package_path_or_metadata_tuples
outputs = get_output_file_paths([_[2] for _ in metadata_tuples])
if output_id:
matched_outputs = [fnmatch(_, output_id) for _ in outputs]
if len(matched_outputs) > 1:
raise ValueError("Specified --output-id matches more than one output ({}). Please refine your output id so that only "
"a single output is found.".format(matched_outputs))
elif not matched_outputs:
print()
raise ValueError("Specified --output-id did not match any outputs. Available outputs are: {} Please check it and try again".format(outputs))
outputs = matched_outputs
if len(outputs) > 1:
raise ValueError("More than one output found for this recipe ({}). Please use the --output-id argument to filter down "
"to a single output.".format(outputs))

target_metadata = metadata_tuples[outputs.index(matched_outputs[0])][2]
if not path:
path = os.path.join(config.croot, "debug_{}_{}".format())

# create the envs
if test:
create_env
create_env(prefix=os.path.join(path, os.path.basename(config.test_prefix)),
specs_or_actions=target_metadata.get_test_deps(), env=None, config=config, subdir=subdir)
else:
pass
# create_build_envs()
269 changes: 134 additions & 135 deletions conda_build/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1455,45 +1455,7 @@ def build(m, stats, post=None, need_source_download=True, need_reparse_in_env=Fa
"or remove the build/script section in meta.yaml.")
# There is no sense in trying to run an empty build script.
if isfile(build_file) or script:

with utils.path_prepended(m.config.build_prefix):
env = environ.get_dict(m=m)
env["CONDA_BUILD_STATE"] = "BUILD"

# hard-code this because we never want pip's build isolation
# https://github.com/conda/conda-build/pull/2972#discussion_r198290241
#
# Note that pip env "NO" variables are inverted logic.
# PIP_NO_BUILD_ISOLATION=False means don't use build isolation.
#
env["PIP_NO_BUILD_ISOLATION"] = False
# some other env vars to have pip ignore dependencies.
# we supply them ourselves instead.
# See note above about inverted logic on "NO" variables
env["PIP_NO_DEPENDENCIES"] = False
env["PIP_IGNORE_INSTALLED"] = True
# pip's cache directory (PIP_NO_CACHE_DIR) should not be
# disabled as this results in .egg-info rather than
# .dist-info directories being created, see gh-3094

# set PIP_CACHE_DIR to a path in the work dir that does not exist.
env['PIP_CACHE_DIR'] = m.config.pip_cache_dir

work_file = join(m.config.work_dir, 'conda_build.sh')
with open(work_file, 'w') as bf:
for k, v in env.items():
if v:
bf.write('export {0}="{1}"\n'.format(k, v))

if m.config.activate and not m.name() == 'conda':
_write_sh_activation_text(bf, m)
if script:
bf.write(script)
if isfile(build_file) and not script:
bf.write(open(build_file).read())

os.chmod(work_file, 0o766)

work_file = write_build_scripts(m, script, build_file)
cmd = [shell_path] + (['-x'] if m.config.debug else []) + ['-e', work_file]

# this should raise if any problems occur while building
Expand Down Expand Up @@ -1878,6 +1840,136 @@ def construct_metadata_for_test(recipedir_or_package, config):
return m, hash_input


def write_build_scripts(m, script, build_file):
with utils.path_prepended(m.config.build_prefix):
env = environ.get_dict(m=m)
env["CONDA_BUILD_STATE"] = "BUILD"

# hard-code this because we never want pip's build isolation
# https://github.com/conda/conda-build/pull/2972#discussion_r198290241
#
# Note that pip env "NO" variables are inverted logic.
# PIP_NO_BUILD_ISOLATION=False means don't use build isolation.
#
env["PIP_NO_BUILD_ISOLATION"] = False
# some other env vars to have pip ignore dependencies.
# we supply them ourselves instead.
# See note above about inverted logic on "NO" variables
env["PIP_NO_DEPENDENCIES"] = False
env["PIP_IGNORE_INSTALLED"] = True
# pip's cache directory (PIP_NO_CACHE_DIR) should not be
# disabled as this results in .egg-info rather than
# .dist-info directories being created, see gh-3094

# set PIP_CACHE_DIR to a path in the work dir that does not exist.
env['PIP_CACHE_DIR'] = m.config.pip_cache_dir

work_file = join(m.config.work_dir, 'conda_build.sh')
env_file = join(m.config.work_dir, 'build_env_setup.sh')
with open(env_file, 'w') as bf:
for k, v in env.items():
if v:
bf.write('export {0}="{1}"\n'.format(k, v))

if m.config.activate and not m.name() == 'conda':
_write_sh_activation_text(bf, m)
with open(work_file, 'w') as bf:
bf.write("source {}\n".format(env_file))
if script:
bf.write(script)
if isfile(build_file) and not script:
bf.write(open(build_file).read())

os.chmod(work_file, 0o766)
return work_file


def write_test_scripts(metadata, env_vars, py_files, pl_files, lua_files, r_files, shell_files, trace=""):
log = utils.get_logger(__name__)
if not metadata.config.activate or metadata.name() == 'conda':
# prepend bin (or Scripts) directory
env_vars = utils.prepend_bin_path(env_vars, metadata.config.test_prefix, prepend_prefix=True)
if utils.on_win:
env_vars['PATH'] = metadata.config.test_prefix + os.pathsep + env_vars['PATH']

# set variables like CONDA_PY in the test environment
env_vars.update(set_language_env_vars(metadata.config.variant))

# Python 2 Windows requires that envs variables be string, not unicode
env_vars = {str(key): str(value) for key, value in env_vars.items()}
suffix = "bat" if utils.on_win else "sh"
test_env_script = join(metadata.config.test_dir,
"conda_test_env_vars.{suffix}".format(suffix=suffix))
test_run_script = join(metadata.config.test_dir,
"conda_test_runner.{suffix}".format(suffix=suffix))

with open(test_env_script, 'w') as tf:
if not utils.on_win:
tf.write('set {trace}-e\n'.format(trace=trace))
if metadata.config.activate and not metadata.name() == 'conda':
ext = ".bat" if utils.on_win else ""
tf.write('{source} "{conda_root}activate{ext}" "{test_env}"\n'.format(
conda_root=utils.root_script_dir + os.path.sep,
source="call" if utils.on_win else "source",
ext=ext,
test_env=metadata.config.test_prefix))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")

with open(test_run_script, 'w') as tf:
ext = ".bat" if utils.on_win else ""
tf.write('{source} "{test_env_script}"\n'.format(
source="call" if utils.on_win else "source",
test_env_script=test_env_script))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if py_files:
test_python = metadata.config.test_python
# use pythonw for import tests when osx_is_app is set
if metadata.get_value('build/osx_is_app') and sys.platform == 'darwin':
test_python = test_python + 'w'
tf.write('"{python}" -s "{test_file}"\n'.format(
python=test_python,
test_file=join(metadata.config.test_dir, 'run_test.py')))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if pl_files:
tf.write('"{perl}" "{test_file}"\n'.format(
perl=metadata.config.perl_bin(metadata.config.test_prefix,
metadata.config.host_platform),
test_file=join(metadata.config.test_dir, 'run_test.pl')))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if lua_files:
tf.write('"{lua}" "{test_file}"\n'.format(
lua=metadata.config.lua_bin(metadata.config.test_prefix,
metadata.config.host_platform),
test_file=join(metadata.config.test_dir, 'run_test.lua')))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if r_files:
tf.write('"{r}" "{test_file}"\n'.format(
r=metadata.config.rscript_bin(metadata.config.test_prefix,
metadata.config.host_platform),
test_file=join(metadata.config.test_dir, 'run_test.r')))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if shell_files:
for shell_file in shell_files:
if utils.on_win:
if os.path.splitext(shell_file) == ".bat":
tf.write('call "{test_file}"\n'.format(test_file=shell_file))
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
else:
log.warn("Found sh test file on windows. Ignoring this for now (PRs welcome)")
else:
# TODO: Run the test/commands here instead of in run_test.py
tf.write('"{shell_path}" {trace}-e "{test_file}"\n'.format(shell_path=shell_path,
test_file=shell_file,
trace=trace))
return test_run_script


def test(recipedir_or_package_or_metadata, config, stats, move_broken=True):
'''
Execute any test scripts for the given package.
Expand Down Expand Up @@ -1955,28 +2047,8 @@ def test(recipedir_or_package_or_metadata, config, stats, move_broken=True):
"in the work directory that are not included with your package")

get_build_metadata(metadata)
specs = ['%s %s %s' % (metadata.name(), metadata.version(), metadata.build_id())]

# add packages listed in the run environment and test/requires
specs.extend(ms.spec for ms in metadata.ms_depends('run'))
specs += utils.ensure_list(metadata.get_value('test/requires', []))

if py_files:
# as the tests are run by python, ensure that python is installed.
# (If they already provided python as a run or test requirement,
# this won't hurt anything.)
specs += ['python']
if pl_files:
# as the tests are run by perl, we need to specify it
specs += ['perl']
if lua_files:
# not sure how this shakes out
specs += ['lua']
if r_files and not any(s.split()[0] in ('r-base', 'mro-base') for s in specs):
# not sure how this shakes out
specs += ['r-base']

specs.extend(utils.ensure_list(metadata.config.extra_deps))

specs = metadata.get_test_deps(py_files, pl_files, lua_files, r_files)

with utils.path_prepended(metadata.config.test_prefix):
env = dict(os.environ.copy())
Expand All @@ -1992,10 +2064,6 @@ def test(recipedir_or_package_or_metadata, config, stats, move_broken=True):
if utils.on_win:
env['PATH'] = metadata.config.test_prefix + os.pathsep + env['PATH']

suffix = "bat" if utils.on_win else "sh"
test_script = join(metadata.config.test_dir,
"conda_test_runner.{suffix}".format(suffix=suffix))

# In the future, we will need to support testing cross compiled
# packages on physical hardware. until then it is expected that
# something like QEMU or Wine will be used on the build machine,
Expand Down Expand Up @@ -2049,77 +2117,8 @@ def test(recipedir_or_package_or_metadata, config, stats, move_broken=True):
if metadata.config.remove_work_dir:
env['SRC_DIR'] = metadata.config.test_dir

if not metadata.config.activate or metadata.name() == 'conda':
# prepend bin (or Scripts) directory
env = utils.prepend_bin_path(env, metadata.config.test_prefix, prepend_prefix=True)
if utils.on_win:
env['PATH'] = metadata.config.test_prefix + os.pathsep + env['PATH']

# set variables like CONDA_PY in the test environment
env.update(set_language_env_vars(metadata.config.variant))

# Python 2 Windows requires that envs variables be string, not unicode
env = {str(key): str(value) for key, value in env.items()}
suffix = "bat" if utils.on_win else "sh"
test_script = join(metadata.config.test_dir,
"conda_test_runner.{suffix}".format(suffix=suffix))
test_script = write_test_scripts(metadata, env, py_files, pl_files, lua_files, r_files, shell_files, trace)

with open(test_script, 'w') as tf:
if not utils.on_win:
tf.write('set {trace}-e\n'.format(trace=trace))
if metadata.config.activate and not metadata.name() == 'conda':
ext = ".bat" if utils.on_win else ""
tf.write('{source} "{conda_root}activate{ext}" "{test_env}"\n'.format(
conda_root=utils.root_script_dir + os.path.sep,
source="call" if utils.on_win else "source",
ext=ext,
test_env=metadata.config.test_prefix))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if py_files:
test_python = metadata.config.test_python
# use pythonw for import tests when osx_is_app is set
if metadata.get_value('build/osx_is_app') and sys.platform == 'darwin':
test_python = test_python + 'w'
tf.write('"{python}" -s "{test_file}"\n'.format(
python=test_python,
test_file=join(metadata.config.test_dir, 'run_test.py')))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if pl_files:
tf.write('"{perl}" "{test_file}"\n'.format(
perl=metadata.config.perl_bin(metadata.config.test_prefix,
metadata.config.host_platform),
test_file=join(metadata.config.test_dir, 'run_test.pl')))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if lua_files:
tf.write('"{lua}" "{test_file}"\n'.format(
lua=metadata.config.lua_bin(metadata.config.test_prefix,
metadata.config.host_platform),
test_file=join(metadata.config.test_dir, 'run_test.lua')))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if r_files:
tf.write('"{r}" "{test_file}"\n'.format(
r=metadata.config.rscript_bin(metadata.config.test_prefix,
metadata.config.host_platform),
test_file=join(metadata.config.test_dir, 'run_test.r')))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if shell_files:
for shell_file in shell_files:
if utils.on_win:
if os.path.splitext(shell_file) == ".bat":
tf.write('call "{test_file}"\n'.format(test_file=shell_file))
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
else:
log.warn("Found sh test file on windows. Ignoring this for now (PRs welcome)")
else:
# TODO: Run the test/commands here instead of in run_test.py
tf.write('"{shell_path}" {trace}-e "{test_file}"\n'.format(shell_path=shell_path,
test_file=shell_file,
trace=trace))
if utils.on_win:
cmd = [os.environ.get('COMSPEC', 'cmd.exe'), "/d", "/c", test_script]
else:
Expand Down
25 changes: 25 additions & 0 deletions conda_build/metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2240,3 +2240,28 @@ def get_top_level_recipe_without_outputs(self):
template_string=recipe_no_outputs,
alt_name=alt_name))
return top_no_outputs or {}

def get_test_deps(self, py_files, pl_files, lua_files, r_files):
specs = ['%s %s %s' % (self.name(), self.version(), self.build_id())]

# add packages listed in the run environment and test/requires
specs.extend(ms.spec for ms in self.ms_depends('run'))
specs += utils.ensure_list(self.get_value('test/requires', []))

if py_files:
# as the tests are run by python, ensure that python is installed.
# (If they already provided python as a run or test requirement,
# this won't hurt anything.)
specs += ['python']
if pl_files:
# as the tests are run by perl, we need to specify it
specs += ['perl']
if lua_files:
# not sure how this shakes out
specs += ['lua']
if r_files and not any(s.split()[0] in ('r-base', 'mro-base') for s in specs):
# not sure how this shakes out
specs += ['r-base']

specs.extend(utils.ensure_list(self.config.extra_deps))
return specs
Loading

0 comments on commit 1ea3af4

Please sign in to comment.