Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Skeleton fixes #2614

Merged
merged 3 commits into from
Jan 2, 2018
Merged

Skeleton fixes #2614

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
554085d
Respect / prefer host binaries
mingwandroid Dec 8, 2017
769f39d
skeleton cran: Split build and host deps
mingwandroid Jan 1, 2018
a4db0c7
skeleton cran: Fix latest tag detection
mingwandroid Jan 2, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions conda_build/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1712,19 +1712,19 @@ def test(recipedir_or_package_or_metadata, config, move_broken=True):
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if pl_files:
tf.write('"{perl}" "{test_file}"\n'.format(
perl=metadata.config.perl_bin(metadata.config.test_prefix),
perl=metadata.config.perl_bin(metadata.config.test_prefix, metadata.config.host_platform),
test_file=join(metadata.config.test_dir, 'run_test.pl')))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if lua_files:
tf.write('"{lua}" "{test_file}"\n'.format(
lua=metadata.config.lua_bin(metadata.config.test_prefix),
lua=metadata.config.lua_bin(metadata.config.test_prefix, metadata.config.host_platform),
test_file=join(metadata.config.test_dir, 'run_test.lua')))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
if r_files:
tf.write('"{r}" CMD BATCH "{test_file}"\n'.format(
r=metadata.config.r_bin(metadata.config.test_prefix),
r=metadata.config.r_bin(metadata.config.test_prefix, metadata.config.host_platform),
test_file=join(metadata.config.test_dir, 'run_test.r')))
if utils.on_win:
tf.write("IF %ERRORLEVEL% NEQ 0 exit 1\n")
Expand Down
41 changes: 22 additions & 19 deletions conda_build/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,8 +410,8 @@ def CONDA_R(self):
def CONDA_R(self, value):
self.variant['r_base'] = value

def _get_python(self, prefix):
if sys.platform == 'win32':
def _get_python(self, prefix, platform):
if platform.startswith('win'):
if os.path.isfile(os.path.join(prefix, 'python_d.exe')):
res = join(prefix, 'python_d.exe')
else:
Expand All @@ -420,26 +420,29 @@ def _get_python(self, prefix):
res = join(prefix, 'bin/python')
return res

def _get_perl(self, prefix):
if sys.platform == 'win32':
def _get_perl(self, prefix, platform):
if platform.startswith('win'):
res = join(prefix, 'Library', 'bin', 'perl.exe')
else:
res = join(prefix, 'bin/perl')
return res

# TODO: This is probably broken on Windows, but no one has a lua package on windows to test.
def _get_lua(self, prefix):
def _get_lua(self, prefix, platform):
lua_ver = self.variant.get('lua', get_default_variant(self)['lua'])
binary_name = "luajit" if (lua_ver and lua_ver[0] == "2") else "lua"
if sys.platform == 'win32':
if platform.startswith('win'):
res = join(prefix, 'Library', 'bin', '{}.exe'.format(binary_name))
else:
res = join(prefix, 'bin/{}'.format(binary_name))
return res

def _get_r(self, prefix):
if sys.platform == 'win32':
def _get_r(self, prefix, platform):
if platform.startswith('win'):
res = join(prefix, 'Scripts', 'R.exe')
# MRO test:
if not os.path.exists(res):
res = join(prefix, 'bin', 'R.exe')
else:
res = join(prefix, 'bin', 'R')
return res
Expand Down Expand Up @@ -537,27 +540,27 @@ def test_prefix(self):

@property
def build_python(self):
return self.python_bin(self.build_prefix)
return self.python_bin(self.build_prefix, self.platform)

@property
def host_python(self):
return self._get_python(self.host_prefix)
return self._get_python(self.host_prefix, self.host_platform)

@property
def test_python(self):
return self.python_bin(self.test_prefix)
return self.python_bin(self.test_prefix, self.host_platform)

def python_bin(self, prefix):
return self._get_python(prefix)
def python_bin(self, prefix, platform):
return self._get_python(prefix, platform)

def perl_bin(self, prefix):
return self._get_perl(prefix)
def perl_bin(self, prefix, platform):
return self._get_perl(prefix, platform)

def lua_bin(self, prefix):
return self._get_lua(prefix)
def lua_bin(self, prefix, platform):
return self._get_lua(prefix, platform)

def r_bin(self, prefix):
return self._get_r(prefix)
def r_bin(self, prefix, platform):
return self._get_r(prefix, platform)

@property
def info_dir(self):
Expand Down
28 changes: 14 additions & 14 deletions conda_build/environ.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,10 +247,10 @@ def get_dict(config, m=None, prefix=None, for_env=True, skip_build_id=False):
d = conda_build_vars(prefix, config)

# languages
d.update(python_vars(config, prefix))
d.update(perl_vars(config, prefix))
d.update(lua_vars(config, prefix))
d.update(r_vars(config, prefix))
d.update(python_vars(config, prefix, config.host_platform))
d.update(perl_vars(config, prefix, config.host_platform))
d.update(lua_vars(config, prefix, config.host_platform))
d.update(r_vars(config, prefix, config.host_platform))

if m:
d.update(meta_vars(m, config, skip_build_id=skip_build_id))
Expand Down Expand Up @@ -299,7 +299,7 @@ def conda_build_vars(prefix, config):
}


def python_vars(config, prefix):
def python_vars(config, prefix, platform):
py_ver = get_py_ver(config)
vars_ = {
'CONDA_PY': ''.join(py_ver.split('.')[:2]),
Expand All @@ -308,9 +308,9 @@ def python_vars(config, prefix):
'STDLIB_DIR': utils.get_stdlib_dir(prefix, py_ver),
'SP_DIR': utils.get_site_packages(prefix, py_ver),
}
if os.path.isfile(config.python_bin(prefix)):
if os.path.isfile(config.python_bin(prefix, platform)):
vars_.update({
'PYTHON': config.python_bin(prefix),
'PYTHON': config.python_bin(prefix, platform),
})

np_ver = config.variant.get('numpy', get_default_variant(config)['numpy'])
Expand All @@ -319,24 +319,24 @@ def python_vars(config, prefix):
return vars_


def perl_vars(config, prefix):
def perl_vars(config, prefix, platform):
vars_ = {
'PERL_VER': get_perl_ver(config),
'CONDA_PERL': get_perl_ver(config),
}
if os.path.isfile(config.perl_bin(prefix)):
if os.path.isfile(config.perl_bin(prefix, platform)):
vars_.update({
'PERL': config.perl_bin(prefix),
'PERL': config.perl_bin(prefix, platform),
})
return vars_


def lua_vars(config, prefix):
def lua_vars(config, prefix, platform):
vars_ = {
'LUA_VER': get_lua_ver(config),
'CONDA_LUA': get_lua_ver(config),
}
lua = config.lua_bin(prefix)
lua = config.lua_bin(prefix, platform)
if os.path.isfile(lua):
vars_.update({
'LUA': lua,
Expand All @@ -345,12 +345,12 @@ def lua_vars(config, prefix):
return vars_


def r_vars(config, prefix):
def r_vars(config, prefix, platform):
vars_ = {
'R_VER': get_r_ver(config),
'CONDA_R': get_r_ver(config),
}
r = config.r_bin(prefix)
r = config.r_bin(prefix, platform)
if os.path.isfile(r):
vars_.update({
'R': r,
Expand Down
107 changes: 62 additions & 45 deletions conda_build/skeletons/cran.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@
requirements:
build:{build_depends}

host:{host_depends}

run:{run_depends}

test:
Expand Down Expand Up @@ -159,19 +161,19 @@
# between versions.
R_BASE_PACKAGE_NAMES = (
'base',
'tools',
'utils',
'grDevices',
'graphics',
'stats',
'compiler',
'datasets',
'methods',
'graphics',
'grDevices',
'grid',
'methods',
'parallel',
'splines',
'stats',
'stats4',
'tcltk',
'compiler',
'parallel',
'tools',
'utils',
)

R_RECOMMENDED_PACKAGE_NAMES = (
Expand Down Expand Up @@ -417,8 +419,13 @@ def get_package_metadata(cran_url, package, session):


def get_latest_git_tag(config):
p = subprocess.Popen(['git', 'describe', '--tags', '--abbrev=0'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=config.work_dir)
# SO says to use taggerdate instead of committerdate, but that is invalid for lightweight tags.
p = subprocess.Popen(['git', 'for-each-ref',
'refs/tags',
'--sort=-committerdate',
'--format=%(refname:short)',
'--count=1'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=config.work_dir)

stdout, stderr = p.communicate()
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
Expand Down Expand Up @@ -675,6 +682,7 @@ def skeletonize(in_packages, output_dir=".", output_suffix="", add_maintainer=No
'patches': '',
'build_number': 0,
'build_depends': '',
'host_depends': '',
'run_depends': '',
# CRAN doesn't seem to have this metadata :(
'home_comment': '#',
Expand Down Expand Up @@ -851,7 +859,14 @@ def skeletonize(in_packages, output_dir=".", output_suffix="", add_maintainer=No
for f in tf])
else:
need_c = need_cxx = need_f = need_autotools = need_make = False
for dep_type in ['build', 'run']:

if 'Rcpp' in dep_dict or 'RcppArmadillo' in dep_dict:
need_cxx = True

if need_cxx:
need_c = True

for dep_type in ['build', 'host', 'run']:

deps = []
# Put non-R dependencies first.
Expand Down Expand Up @@ -879,7 +894,8 @@ def skeletonize(in_packages, output_dir=".", output_suffix="", add_maintainer=No
deps.append("{indent}{{{{posix}}}}grep # [win]".format(
indent=INDENT))
deps.append("{indent}{{{{posix}}}}autoconf".format(indent=INDENT))
deps.append("{indent}{{{{posix}}}}automake".format(indent=INDENT))
deps.append("{indent}{{{{posix}}}}automake-wrapper # [win]".format(indent=INDENT))
deps.append("{indent}{{{{posix}}}}automake # [not win]".format(indent=INDENT))
deps.append("{indent}{{{{posix}}}}pkg-config".format(indent=INDENT))
if need_make:
deps.append("{indent}{{{{posix}}}}make".format(indent=INDENT))
Expand All @@ -888,39 +904,40 @@ def skeletonize(in_packages, output_dir=".", output_suffix="", add_maintainer=No
deps.append("{indent}{{{{native}}}}gcc-libs # [win]".format(
indent=INDENT))

for name in sorted(dep_dict):
if name in R_BASE_PACKAGE_NAMES:
continue
if name == 'R':
# Put R first
# Regarless of build or run, and whether this is a recommended package or not,
# it can only depend on 'r-base' since anything else can and will cause cycles
# in the dependency graph. The cran metadata lists all dependencies anyway, even
# those packages that are in the recommended group.
r_name = 'r-base'
# We don't include any R version restrictions because we
# always build R packages against an exact R version
deps.insert(0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_name))
else:
conda_name = 'r-' + name.lower()

if dep_dict[name]:
deps.append('{indent}{name} {version}'.format(name=conda_name,
version=dep_dict[name], indent=INDENT))
if dep_type == 'host' or dep_type == 'run':
for name in sorted(dep_dict):
if name in R_BASE_PACKAGE_NAMES:
continue
if name == 'R':
# Put R first
# Regarless of build or run, and whether this is a recommended package or not,
# it can only depend on 'r-base' since anything else can and will cause cycles
# in the dependency graph. The cran metadata lists all dependencies anyway, even
# those packages that are in the recommended group.
r_name = 'r-base'
# We don't include any R version restrictions because we
# always build R packages against an exact R version
deps.insert(0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_name))
else:
deps.append('{indent}{name}'.format(name=conda_name,
indent=INDENT))
if recursive:
lower_name = name.lower()
if lower_name not in package_dicts:
inputs_dict = package_to_inputs_dict(output_dir, output_suffix, git_tag,
lower_name)
assert lower_name == inputs_dict['pkg-name'], \
"name %s != inputs_dict['pkg-name'] %s" % (name,
inputs_dict['pkg-name'])
assert lower_name not in package_list
package_dicts.update({lower_name: {'inputs': inputs_dict}})
package_list.append(lower_name)
conda_name = 'r-' + name.lower()

if dep_dict[name]:
deps.append('{indent}{name} {version}'.format(name=conda_name,
version=dep_dict[name], indent=INDENT))
else:
deps.append('{indent}{name}'.format(name=conda_name,
indent=INDENT))
if recursive:
lower_name = name.lower()
if lower_name not in package_dicts:
inputs_dict = package_to_inputs_dict(output_dir, output_suffix, git_tag,
lower_name)
assert lower_name == inputs_dict['pkg-name'], \
"name %s != inputs_dict['pkg-name'] %s" % (name,
inputs_dict['pkg-name'])
assert lower_name not in package_list
package_dicts.update({lower_name: {'inputs': inputs_dict}})
package_list.append(lower_name)

d['%s_depends' % dep_type] = ''.join(deps)

Expand All @@ -939,7 +956,7 @@ def skeletonize(in_packages, output_dir=".", output_suffix="", add_maintainer=No
elif update_policy == 'skip-existing' and d['inputs']['old-metadata']:
continue

# Normalize the metadata values
# Normalize the metadata values
d = {k: unicodedata.normalize("NFKD", text_type(v)).encode('ascii', 'ignore')
.decode() for k, v in iteritems(d)}
try:
Expand Down