Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/environments #63

Closed
wants to merge 8 commits into from
Closed

Feature/environments #63

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c98d3f1
Apply changes from environment branch
adrienbernede Apr 22, 2021
9f45ce9
Explicit indexing in strings args
adrienbernede Apr 23, 2021
475791e
trailing whitespaces
adrienbernede Apr 23, 2021
fb16e90
Fix renaming in recent version
adrienbernede Apr 23, 2021
38847ba
Adds TODO
adrienbernede Apr 23, 2021
d4e39b2
Allow both 'patch' and 'env' modes
adrienbernede May 4, 2021
3751965
Merge branch 'master' into feature/environments
adrienbernede Sep 1, 2021
6bb7346
fix
adrienbernede Sep 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 77 additions & 29 deletions uberenv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@

import os
import sys
import stat
import subprocess
import shutil
import socket
Expand Down Expand Up @@ -137,7 +138,7 @@ def parse_args():
parser.add_option("--spack-config-dir",
dest="spack_config_dir",
default=None,
help="dir with spack settings files (compilers.yaml, packages.yaml, etc)")
help="dir with spack settings files (spack.yaml, config.yaml, compilers.yaml, packages.yaml, etc)")

# this option allows a user to set the directory for their vcpkg ports on Windows
parser.add_option("--vcpkg-ports-path",
Expand All @@ -151,6 +152,14 @@ def parse_args():
default=None,
help="override the default package name")

# spack configuration mode
parser.add_option("--spack-config-mode",
dest="spack_config_mode",
default=None,
help="set mode used to apply a config to spack"
"(options: 'patch' 'env'"
"[default: 'patch'] )\n")

# uberenv spack tpl build mode
parser.add_option("--spack-build-mode",
dest="spack_build_mode",
Expand Down Expand Up @@ -339,7 +348,7 @@ def setup_paths_and_dirs(self):

def set_from_args_or_json(self,setting, optional=True):
"""
When optional=False:
When optional=False:
If the setting key is not in the json file, error and raise an exception.
When optional=True:
If the setting key is not in the json file or opts, return None.
Expand All @@ -357,7 +366,7 @@ def set_from_args_or_json(self,setting, optional=True):

def set_from_json(self,setting, optional=True):
"""
When optional=False:
When optional=False:
If the setting key is not in the json file, error and raise an exception.
When optional=True:
If the setting key is not in the json file or opts, return None.
Expand Down Expand Up @@ -390,7 +399,7 @@ def __init__(self, opts, extra_opts):

# setup architecture triplet
self.vcpkg_triplet = self.set_from_args_or_json("vcpkg_triplet")
print("Vcpkg triplet: {}".format(self.vcpkg_triplet))
print("Vcpkg triplet: {0}".format(self.vcpkg_triplet))
if self.vcpkg_triplet is None:
self.vcpkg_triplet = os.getenv("VCPKG_DEFAULT_TRIPLET", "x86-windows")

Expand Down Expand Up @@ -442,7 +451,7 @@ def clone_repo(self):

os.chdir(self.dest_dir)

clone_opts = ("-c http.sslVerify=false "
clone_opts = ("-c http.sslVerify=false "
if self.opts["ignore_ssl_errors"] else "")

clone_cmd = "git {0} clone --single-branch -b {1} {2} vcpkg".format(clone_opts, vcpkg_branch,vcpkg_url)
Expand All @@ -454,7 +463,7 @@ def clone_repo(self):
print("[info: using vcpkg commit {0}]".format(sha1))
os.chdir(self.dest_vcpkg)
sexe("git checkout {0}".format(sha1),echo=True)

if self.opts["repo_pull"]:
# do a pull to make sure we have the latest
os.chdir(self.dest_vcpkg)
Expand All @@ -473,7 +482,7 @@ def clone_repo(self):

def patch(self):
""" hot-copy our ports into vcpkg """

import distutils.dir_util

dest_vcpkg_ports = pjoin(self.dest_vcpkg, "ports")
Expand All @@ -500,7 +509,7 @@ def use_mirror(self):
pass

def install(self):

os.chdir(self.dest_vcpkg)
install_cmd = "vcpkg.exe "
install_cmd += "install {0}:{1}".format(self.pkg_name, self.vcpkg_triplet)
Expand All @@ -523,11 +532,18 @@ class SpackEnv(UberEnv):

def __init__(self, opts, extra_opts):
UberEnv.__init__(self,opts,extra_opts)

self.spack_cmd = "spack/bin/spack"

self.pkg_version = self.set_from_json("package_version")
self.pkg_src_dir = self.set_from_args_or_json("package_source_dir", True)
self.pkg_final_phase = self.set_from_args_or_json("package_final_phase",True)
# get build mode
self.build_mode = self.set_from_args_or_json("spack_build_mode",True)
self.spack_config_mode = self.set_from_args_or_json("spack_config_mode",True)
# default spack config mode is "patch" (alternative: "env")
if self.spack_config_mode is None:
self.spack_config_mode = "patch"
# default spack build mode is dev-build
if self.build_mode is None:
self.build_mode = "dev-build"
Expand Down Expand Up @@ -647,15 +663,15 @@ def setup_paths_and_dirs(self):
sys.exit(-1)

def find_spack_pkg_path_from_hash(self, pkg_name, pkg_hash):
res, out = sexe("spack/bin/spack find -p /{0}".format(pkg_hash), ret_output = True)
res, out = sexe("{0} find -p /{1}".format(self.spack_cmd, pkg_hash), ret_output = True)
for l in out.split("\n"):
if l.startswith(pkg_name):
return {"name": pkg_name, "path": l.split()[-1]}
print("[ERROR: failed to find package named '{0}']".format(pkg_name))
sys.exit(-1)

def find_spack_pkg_path(self, pkg_name, spec = ""):
res, out = sexe("spack/bin/spack find -p " + pkg_name + spec,ret_output = True)
res, out = sexe("{0} find -p {1}".format(self.spack_cmd, pkg_name + spec), ret_output = True)
for l in out.split("\n"):
# TODO: at least print a warning when several choices exist. This will
# pick the first in the list.
Expand All @@ -666,7 +682,7 @@ def find_spack_pkg_path(self, pkg_name, spec = ""):

# Extract the first line of the full spec
def read_spack_full_spec(self,pkg_name,spec):
res, out = sexe("spack/bin/spack spec " + pkg_name + " " + spec, ret_output=True)
res, out = sexe("{0} spec {1} {2}".format(self.spack_cmd, pkg_name, spec), ret_output=True)
for l in out.split("\n"):
if l.startswith(pkg_name) and l.count("@") > 0 and l.count("arch=") > 0:
return l.strip()
Expand Down Expand Up @@ -713,6 +729,7 @@ def clone_repo(self):
print("[ERROR: Git failed to pull]")
sys.exit(-1)

#IF Old patching mode
def disable_spack_config_scopes(self,spack_dir):
# disables all config scopes except "defaults", which we will
# force our settings into
Expand Down Expand Up @@ -740,15 +757,15 @@ def patch(self):
spack_etc_defaults_dir = pjoin(spack_dir,"etc","spack","defaults")

if cfg_dir is not None:
print("[copying spack settings from {0}]".format(cfg_dir))

# copy in "defaults" config.yaml
config_yaml = pabs(pjoin(cfg_dir,"..","config.yaml"))
sexe("cp {0} {1}/".format(config_yaml, spack_etc_defaults_dir), echo=True)
mirrors_yaml = pabs(pjoin(cfg_dir,"..","mirrors.yaml"))
sexe("cp {0} {1}/".format(mirrors_yaml, spack_etc_defaults_dir), echo=True)

# copy in other settings per platform
print("[copying uberenv compiler and packages settings from {0}]".format(cfg_dir))

config_yaml = pjoin(cfg_dir,"config.yaml")
mirrors_yaml = pjoin(cfg_dir,"mirrors.yaml")
compilers_yaml = pjoin(cfg_dir,"compilers.yaml")
Expand All @@ -769,17 +786,45 @@ def patch(self):
# let spack try to auto find compilers
sexe("spack/bin/spack compiler find", echo=True)

# ELSE new environment mode
def load(self):
# load spack environment, potentially overriding spack defaults
# uberenv used to handle defaults overriding, now spack environment can
# be used to do so, but it lies with each project to make sure their
# environment is well designed.

lock_file=self.spack_config_dir+"/spack.lock"
sexe("[[ -e ${0} ]] && rm ${0}".format(lock_file)

self.spack_cmd = "{0} -e {1}".format(self.spack_cmd,self.spack_config_dir)

# A simple proxy script for spack
uber_spack='#!/bin/bash\n$(dirname ${{0}})/{0} "$@"\n'.format(self.spack_cmd)

with open('uber-spack','w+') as script:
script.write(uber_spack)

# Making the script executable
st = os.stat('uber-spack')
os.chmod('uber-spack', st.st_mode | stat.S_IEXEC)

# this is an opportunity to show spack python info post obtaining spack
self.print_spack_python_info()

# TODO: The hot copy may not be necessary any more, but we will deal with this
# after we implement the copy of environments and mirror setting.
# note: the feature to use here is: spack repo
# hot-copy our packages into spack
if len(self.packages_paths) > 0:
dest_spack_pkgs = pjoin(spack_dir,"var","spack","repos","builtin","packages")
dest_spack_pkgs = pjoin(self.dest_spack,"var","spack","repos","builtin","packages")
for _base_path in self.packages_paths:
_src_glob = pjoin(_base_path, "*")
print("[copying patched packages from {0}]".format(_src_glob))
sexe("cp -Rf {0} {1}".format(_src_glob, dest_spack_pkgs))

# Update spack's config.yaml if clingo was requested
if self.use_clingo:
concretizer_cmd = "spack/bin/spack config --scope defaults add config:concretizer:clingo"
concretizer_cmd = "{0} config --scope defaults add config:concretizer:clingo".format(self.spack_cmd)
res = sexe(concretizer_cmd, echo=True)
if res != 0:
print("[ERROR: Failed to update spack configuration to use new concretizer]")
Expand All @@ -790,22 +835,22 @@ def patch(self):
def clean_build(self):
# clean out any spack cached stuff (except build stages, downloads, &
# spack's bootstrapping software)
cln_cmd = "spack/bin/spack clean --misc-cache --failures --python-cache"
cln_cmd = "{0} clean --misc-cache --failures --python-cache".format(self.spack_cmd)
res = sexe(cln_cmd, echo=True)

# check if we need to force uninstall of selected packages
if self.opts["spack_clean"]:
if self.project_opts.has_key("spack_clean_packages"):
for cln_pkg in self.project_opts["spack_clean_packages"]:
if self.find_spack_pkg_path(cln_pkg) is not None:
unist_cmd = "spack/bin/spack uninstall -f -y --all --dependents " + cln_pkg
res = sexe(unist_cmd, echo=True)
uninst_cmd = "{0} uninstall -f -y --all --dependents ".format(self.spack_cmd) + cln_pkg
res = sexe(uninst_cmd, echo=True)

def show_info(self):
# print concretized spec with install info
# default case prints install status and 32 characters hash
options="--install-status --very-long"
spec_cmd = "spack/bin/spack spec {0} {1}{2}".format(options,self.pkg_name,self.opts["spec"])
spec_cmd = "{0} spec {1} {2}{3}".format(self.spack_cmd,options,self.pkg_name,self.opts["spec"])

res, out = sexe(spec_cmd, ret_output=True, echo=True)
print(out)
Expand Down Expand Up @@ -833,7 +878,7 @@ def install(self):
# use the uberenv package to trigger the right builds
# and build an host-config.cmake file
if not self.use_install:
install_cmd = "spack/bin/spack "
install_cmd = "{0} ".format(self.spack_cmd)
if self.opts["ignore_ssl_errors"]:
install_cmd += "-k "
# build mode -- install path
Expand Down Expand Up @@ -877,19 +922,19 @@ def install(self):
activate=False
break
if activate:
activate_cmd = "spack/bin/spack activate " + pkg_name
activate_cmd = "{0} activate {1}".format(self.spack_cmd,pkg_name)
res = sexe(activate_cmd, echo=True)
if res != 0:
return res
print("[done activating dependent packages]")
# note: this assumes package extends python when +python
# this may fail general cases
if self.build_mode == "install" and "+python" in full_spec:
activate_cmd = "spack/bin/spack activate /" + self.spec_hash
activate_cmd = "{0} activate /{1}".format(self.spack_cmd, self.spec_hash)
res = sexe(activate_cmd, echo=True)
if res != 0:
return res
# when using install or uberenv-pkg mode, create a symlink to the host config
# when using install or uberenv-pkg mode, create a symlink to the host config
if self.build_mode == "install" or \
self.build_mode == "uberenv-pkg" \
or self.use_install:
Expand Down Expand Up @@ -958,7 +1003,7 @@ def create_mirror(self):

mirror_path = self.get_mirror_path()

mirror_cmd = "spack/bin/spack "
mirror_cmd = self.spack_cmd()
if self.opts["ignore_ssl_errors"]:
mirror_cmd += "-k "
mirror_cmd += "mirror create -d {0} --dependencies {1}{2}".format(mirror_path,
Expand All @@ -971,7 +1016,7 @@ def find_spack_mirror(self, mirror_name):
Returns the path of a defaults scoped spack mirror with the
given name, or None if no mirror exists.
"""
res, out = sexe("spack/bin/spack mirror list", ret_output=True)
res, out = sexe("{0} mirror list".format(self.spack_cmd), ret_output=True)
mirror_path = None
for mirror in out.split('\n'):
if mirror:
Expand All @@ -996,12 +1041,12 @@ def use_mirror(self):
# Note: In this case, spack says it removes the mirror, but we still
# get errors when we try to add a new one, sounds like a bug
#
sexe("spack/bin/spack mirror remove --scope=defaults {0} ".format(mirror_name),
sexe("{0} mirror remove --scope=defaults {1} ".format(self.spack_cmd,mirror_name),
echo=True)
existing_mirror_path = None
if not existing_mirror_path:
# Add if not already there
sexe("spack/bin/spack mirror add --scope=defaults {0} {1}".format(
sexe("{1} mirror add --scope=defaults {1} {2}".format(
mirror_name, mirror_path), echo=True)
print("[using mirror {0}]".format(mirror_path))

Expand All @@ -1012,7 +1057,7 @@ def find_spack_upstream(self, upstream_name):
"""
upstream_path = None

res, out = sexe('spack/bin/spack config get upstreams', ret_output=True)
res, out = sexe('{0} config get upstreams'.format(self.spack_cmd), ret_output=True)
if (not out) and ("upstreams:" in out):
out = out.replace(' ', '')
out = out.replace('install_tree:', '')
Expand Down Expand Up @@ -1155,7 +1200,10 @@ def main():
os.chdir(env.dest_dir)

# Patch the package manager, as necessary
env.patch()
if not is_windows() and self.spack_config_mode == "env":
env.load()
else:
env.patch()

# Clean the build
env.clean_build()
Expand Down