From 2aa56727a6d1e15dcf3854c2718ed2062c14fb2b Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Tue, 18 Jan 2022 14:59:39 +0800 Subject: [PATCH 01/38] reverting string to parse config params --- shpc/client/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shpc/client/config.py b/shpc/client/config.py index bef91fe3b..532985ca5 100644 --- a/shpc/client/config.py +++ b/shpc/client/config.py @@ -41,7 +41,8 @@ def main(args, parser, extra, subparser): % param ) continue - key, value = param.split(":", 1) + value, key = param[::-1].split(":", 1) + key, value = key[::-1], value[::-1] if command == "set": cli.settings.set(key, value) logger.info("Updated %s to be %s" % (key, value)) From 3b4058677d482a68262a79d63d818d966682d246 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Tue, 18 Jan 2022 15:45:28 +0800 Subject: [PATCH 02/38] config get: changed behaviour to work with dicts --- shpc/main/settings.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/shpc/main/settings.py b/shpc/main/settings.py index 96549cc37..2934b05d2 100644 --- a/shpc/main/settings.py +++ b/shpc/main/settings.py @@ -114,7 +114,12 @@ def get(self, key, default=None): """ Get a settings value, doing appropriate substitution and expansion. """ - value = self._settings.get(key, default) + # This is a reference to a dictionary (object) setting + if ":" in key: + key, subkey = key.split(":") + value = self._settings[key][subkey] + else: + value = self._settings[key] value = self._substitutions(value) # If we allow environment substitution, do it if key in defaults.allowed_envars and value: From e2b2bf669ea9f62c23431434b24a74b56b437a2a Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Tue, 18 Jan 2022 15:58:44 +0800 Subject: [PATCH 03/38] reverting config get impl for non-dict, to try and fix tests --- shpc/main/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shpc/main/settings.py b/shpc/main/settings.py index 2934b05d2..f57f36f8a 100644 --- a/shpc/main/settings.py +++ b/shpc/main/settings.py @@ -119,7 +119,7 @@ def get(self, key, default=None): key, subkey = key.split(":") value = self._settings[key][subkey] else: - value = self._settings[key] + value = self._settings.get(key, default) value = self._substitutions(value) # If we allow environment substitution, do it if key in defaults.allowed_envars and value: From e809003e40899f92e9a31d17018cbcb39d3b26e0 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Tue, 18 Jan 2022 16:58:33 +0800 Subject: [PATCH 04/38] adding wrapper script variables --- shpc/main/schemas.py | 2 ++ shpc/settings.yml | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/shpc/main/schemas.py b/shpc/main/schemas.py index ae0d7d0a0..291052285 100644 --- a/shpc/main/schemas.py +++ b/shpc/main/schemas.py @@ -124,11 +124,13 @@ "environment_file": {"type": "string"}, "default_version": {"type": "boolean"}, "enable_tty": {"type": "boolean"}, + "wrapper_scripts": {"type": "boolean"}, "container_tech": {"type": "string", "enum": ["singularity", "podman", "docker"]}, "singularity_shell": {"type": "string", "enum": shells}, "podman_shell": {"type": "string", "enum": shells}, "docker_shell": {"type": "string", "enum": shells}, "test_shell": {"type": "string", "enum": shells}, + "wrapper_shell": {"type": "string", "enum": shells}, "module_sys": {"type": "string", "enum": ["lmod", "tcl", None]}, "container_features": container_features, } diff --git a/shpc/settings.yml b/shpc/settings.yml index c89aa78f0..93c58d836 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -19,7 +19,7 @@ registry: [$root_dir/registry] # Lmod or Environment Modules settings # The install directory for modules. Defaults to the install directory/modules -module_base: $root_dir/modules +module_base: $root_dir/../modules # Format string for module commands exec,run,shell (not aliases) (can also include: # {{registry}} , {{ repository }}, {{tool}}, {{version}} @@ -27,13 +27,13 @@ module_base: $root_dir/modules module_name: '{{ tool }}' # Create a .version file for LMOD in the module folder -default_version: true +default_version: false # store containers separately from module files -container_base: +container_base: $root_dir/../containers # if defined, add to lmod script to load this Singularity module first -singularity_module: +singularity_module: singularity podman_module: # string with comma separated list of paths to binds. If set, expored to SINGULARITY_BINDPATH @@ -50,6 +50,12 @@ docker_shell: /bin/sh # shell for test.sh file test_shell: /bin/bash +# for container aliases, use wrapper shellscripts instead of shell aliases, defaults to false +wrapper_scripts: false + +# shell for wrapper shellscripts +wrapper_shell: /bin/bash + # the module namespace you want to install from. E.g., if you have ghcr.io/autamus/clingo # and you set the namespace to ghcr.io/autamus, you can just do: shpc install clingo. namespace: @@ -60,8 +66,9 @@ environment_file: 99-shpc.sh # container features like gpu will modify generated recipes container_features: - gpu: # one of null, amd, or nvidia - x11: # one of null, true, false, or a path + gpu: amd # one of null, amd, or nvidia + x11: true # one of null, true, false, or a path # defaults to ~/.Xauthority if set to true and the container has x11: true - home: # one of null, or a single path or src:dest path. + home: $HOME/.shpc_home # one of null, or a single path or src:dest path. # home: true in a container.yaml will use this path, if defines +updated_at: '2022-01-18T16:13:38Z' From 7ea838d032f4891e5e39c86478da5f41c7766bc9 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Wed, 19 Jan 2022 11:51:36 +0800 Subject: [PATCH 05/38] fix to updated settings.yml --- shpc/settings.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/shpc/settings.yml b/shpc/settings.yml index 93c58d836..290076f94 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -19,7 +19,7 @@ registry: [$root_dir/registry] # Lmod or Environment Modules settings # The install directory for modules. Defaults to the install directory/modules -module_base: $root_dir/../modules +module_base: $root_dir/modules # Format string for module commands exec,run,shell (not aliases) (can also include: # {{registry}} , {{ repository }}, {{tool}}, {{version}} @@ -27,13 +27,13 @@ module_base: $root_dir/../modules module_name: '{{ tool }}' # Create a .version file for LMOD in the module folder -default_version: false +default_version: true # store containers separately from module files -container_base: $root_dir/../containers +container_base: # if defined, add to lmod script to load this Singularity module first -singularity_module: singularity +singularity_module: podman_module: # string with comma separated list of paths to binds. If set, expored to SINGULARITY_BINDPATH @@ -66,9 +66,8 @@ environment_file: 99-shpc.sh # container features like gpu will modify generated recipes container_features: - gpu: amd # one of null, amd, or nvidia - x11: true # one of null, true, false, or a path + gpu: # one of null, amd, or nvidia + x11: # one of null, true, false, or a path # defaults to ~/.Xauthority if set to true and the container has x11: true - home: $HOME/.shpc_home # one of null, or a single path or src:dest path. + home: # one of null, or a single path or src:dest path. # home: true in a container.yaml will use this path, if defines -updated_at: '2022-01-18T16:13:38Z' From a74101753c90275a2db61aa8f2e0a7390dc63d65 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Wed, 19 Jan 2022 14:39:18 +0800 Subject: [PATCH 06/38] new default: containers_base distinct from modules_base --- shpc/settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shpc/settings.yml b/shpc/settings.yml index 290076f94..bdd898509 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -30,7 +30,7 @@ module_name: '{{ tool }}' default_version: true # store containers separately from module files -container_base: +container_base: $root_dir/containers # if defined, add to lmod script to load this Singularity module first singularity_module: From c59bfa74fc575d4f5e852e425d3b23bc84eaf54d Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Wed, 19 Jan 2022 17:32:28 +0800 Subject: [PATCH 07/38] 1st pass at wrapper_scripts for singularity --- shpc/main/container/base.py | 2 +- shpc/main/container/singularity.py | 23 ++++++++++++++++++++- shpc/main/modules/__init__.py | 5 ++++- shpc/main/modules/templates/singularity.lua | 11 +++++++++- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/shpc/main/container/base.py b/shpc/main/container/base.py index 8dde59b33..e646f5413 100644 --- a/shpc/main/container/base.py +++ b/shpc/main/container/base.py @@ -57,7 +57,7 @@ def __init__(self): self.settings = SettingsBase() - def add(self, sif, module_name, modulefile, template, **kwargs): + def add(self, sif, module_name, modulefile, template, wrapper_template, **kwargs): """ Manually add a registry container. """ diff --git a/shpc/main/container/singularity.py b/shpc/main/container/singularity.py index 177351ad6..e3680692a 100644 --- a/shpc/main/container/singularity.py +++ b/shpc/main/container/singularity.py @@ -71,7 +71,7 @@ def get(self, module_name, env_file=False): logger.exit("Found more than one sif in module folder.") return sif[0] - def add(self, sif, module_name, modulefile, template, **kwargs): + def add(self, sif, module_name, modulefile, template, wrapper_template, **kwargs): """ Manually add a registry container. """ @@ -112,6 +112,7 @@ def add(self, sif, module_name, modulefile, template, **kwargs): template, parsed_name=parsed_name, features=kwargs.get("features"), + wrapper_template=wrapper_template, ) self.add_environment(module_dir, {}, self.settings.environment_file) logger.info("Module %s was created." % (module_name)) @@ -130,6 +131,7 @@ def install( config_features=None, features=None, version=None, + wrapper_template=None, ): """Install a general container path to a module @@ -166,6 +168,23 @@ def install( labels = {} logger.warning("Singularity is not installed, skipping metadata.") + # Option to create wrapper scripts for commands + wrapper_dir = os.path.join(container_dir, "bin") + if self.settings.wrapper_scripts and aliases: + shpc.utils.mkdirp([wrapper_dir]) + for alias in aliases: + wrapper_path = os.path.join(wrapper_dir, alias['name']) + out = wrapper_template.render( + alias=alias, + bindpaths=self.settings.bindpaths, + container_sif=container_path, + features=features, + module_dir=os.path.dirname(module_path), + envfile=self.settings.environment_file, + wrapper_shell=self.settings.wrapper_shell, + ) + shpc.utils.write_file(wrapper_path, out) + # Make sure to render all values! out = template.render( singularity_module=self.settings.singularity_module, @@ -186,6 +205,8 @@ def install( registry=parsed_name.registry, repository=parsed_name.repository, envfile=self.settings.environment_file, + wrapper_scripts=self.settings.wrapper_scripts, + wrapper_dir=wrapper_dir, ) shpc.utils.write_file(module_path, out) diff --git a/shpc/main/modules/__init__.py b/shpc/main/modules/__init__.py index 2b040d297..fd0e7e54a 100644 --- a/shpc/main/modules/__init__.py +++ b/shpc/main/modules/__init__.py @@ -147,7 +147,8 @@ def add(self, sif, module_name, **kwargs): module_name = self.add_namespace(module_name) template = self._load_template(self.templatefile) modulefile = os.path.join(self.settings.module_base, module_name.replace(":", os.sep), self.modulefile) - self.container.add(sif, module_name, modulefile, template, **kwargs) + wrapper_template = self._load_template('.'.join([self.container.templatefile,'.sh'])) + self.container.add(sif, module_name, modulefile, template, wrapper_template, **kwargs) def get(self, module_name, env_file=False): """ @@ -325,6 +326,7 @@ def install(self, name, tag=None, **kwargs): # Get the template based on the module and container type template = self._load_template(self.templatefile) module_path = os.path.join(module_dir, self.modulefile) + wrapper_template = self._load_template('.'.join([self.container.templatefile,'sh'])) # If the module has a version, overrides version version = tag.name @@ -344,6 +346,7 @@ def install(self, name, tag=None, **kwargs): version=version, config_features=config.features, features=kwargs.get("features"), + wrapper_template=wrapper_template, ) # If the container tech does not need storage, clean up diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index 060b4eaf9..8b029df9e 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -38,7 +38,7 @@ For each of the above, you can export: {% if singularity_module %}load("{{ singularity_module }}"){% endif %} --- we probably don't need this +-- we probably do not need this local MODULEPATH="{{ module_dir }}" -- singularity environment variables to bind the paths and set shell @@ -60,6 +60,13 @@ set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) -- conflict with modules with the same name conflict(myModuleName(){% if aliases %}{% for alias in aliases %}{% if alias.name != name %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) +{% if wrapper_scripts %} +-- add path with "alias" wrapper scripts to PATH +{% if aliases %} +prepend_path("PATH", "{{ wrapper_dir }}") +{% endif %} + +{% else %} -- exec functions to provide "alias" to module commands {% if aliases %}{% for alias in aliases %} set_shell_function("{{ alias.name }}", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }} $@", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }}") @@ -71,6 +78,8 @@ if (myShellName() == "bash") then {% endfor %} end{% endif %} +{% endif %} + -- A customizable exec function set_shell_function("{|module_name|}-exec", execCmd .. containerPath .. " $@", execCmd .. containerPath) From 8fa3ba15de760268a2cc74d925afc4299bb71b16 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Thu, 20 Jan 2022 10:11:47 +0800 Subject: [PATCH 08/38] wrapper_script: added tcl for singularity --- shpc/main/modules/templates/singularity.lua | 4 +--- shpc/main/modules/templates/singularity.tcl | 7 +++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index 8b029df9e..215c78055 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -62,9 +62,7 @@ conflict(myModuleName(){% if aliases %}{% for alias in aliases %}{% if alias.nam {% if wrapper_scripts %} -- add path with "alias" wrapper scripts to PATH -{% if aliases %} -prepend_path("PATH", "{{ wrapper_dir }}") -{% endif %} +{% if aliases %}prepend_path("PATH", "{{ wrapper_dir }}"){% endif %} {% else %} -- exec functions to provide "alias" to module commands diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl index c24cc0e90..ee5c027b0 100644 --- a/shpc/main/modules/templates/singularity.tcl +++ b/shpc/main/modules/templates/singularity.tcl @@ -74,6 +74,11 @@ set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_ # set_shell_function takes bashStr and cshStr set-alias {|module_name|}-shell "${shellCmd}" +{% if wrapper_scripts %} +# add path with "alias" wrapper scripts to PATH +{% if aliases %}prepend-path PATH "{{ wrapper_dir }}"{% endif %} + +{% else %} # exec functions to provide "alias" to module commands {% if aliases %} if { [ module-info shell bash ] } { @@ -91,6 +96,8 @@ if { [ module-info shell bash ] } { } {% endif %} +{% endif %} + # A customizable exec function set-alias {|module_name|}-exec "${execCmd} ${containerPath}" From bc6b0e8b7c29d8f44c455f8c8607b097247fc493 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Thu, 20 Jan 2022 10:12:36 +0800 Subject: [PATCH 09/38] wrapper script: added script template for real (was in gitignore) --- shpc/main/modules/templates/singularity.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 shpc/main/modules/templates/singularity.sh diff --git a/shpc/main/modules/templates/singularity.sh b/shpc/main/modules/templates/singularity.sh new file mode 100644 index 000000000..2766a917d --- /dev/null +++ b/shpc/main/modules/templates/singularity.sh @@ -0,0 +1,3 @@ +#!{{ wrapper_shell }} + +singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} {% if alias.singularity_options %} {{ alias.singularity_options }} {% endif %} {{ container_sif }} {{ alias.command }} {% if '/sh' in wrapper_shell or '/bash' in wrapper_shell %}$@{% elif '/csh' in wrapper_shell %}$*{% endif %} \ No newline at end of file From 599c44223591b99f5beff8e436b12889d44748d2 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Thu, 20 Jan 2022 10:18:22 +0800 Subject: [PATCH 10/38] wrapper script: fix for add case --- shpc/main/modules/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shpc/main/modules/__init__.py b/shpc/main/modules/__init__.py index f12323b20..765af0da0 100644 --- a/shpc/main/modules/__init__.py +++ b/shpc/main/modules/__init__.py @@ -147,7 +147,7 @@ def add(self, sif, module_name, **kwargs): module_name = self.add_namespace(module_name) template = self._load_template(self.templatefile) modulefile = os.path.join(self.settings.module_base, module_name.replace(":", os.sep), self.modulefile) - wrapper_template = self._load_template('.'.join([self.container.templatefile,'.sh'])) + wrapper_template = self._load_template('.'.join([self.container.templatefile,'sh'])) self.container.add(sif, module_name, modulefile, template, wrapper_template, **kwargs) def get(self, module_name, env_file=False): From 693bdafd5f54da257fb08dfaef0a590db472734f Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Thu, 20 Jan 2022 10:36:17 +0800 Subject: [PATCH 11/38] wrapper scripts now created as executable --- shpc/main/container/singularity.py | 2 +- shpc/main/modules/templates/singularity.sh | 0 shpc/utils/fileio.py | 5 ++++- 3 files changed, 5 insertions(+), 2 deletions(-) mode change 100644 => 100755 shpc/main/modules/templates/singularity.sh diff --git a/shpc/main/container/singularity.py b/shpc/main/container/singularity.py index 6249b282f..87af3de58 100644 --- a/shpc/main/container/singularity.py +++ b/shpc/main/container/singularity.py @@ -183,7 +183,7 @@ def install( envfile=self.settings.environment_file, wrapper_shell=self.settings.wrapper_shell, ) - shpc.utils.write_file(wrapper_path, out) + shpc.utils.write_file(wrapper_path, out, exec=True) # Make sure to render all values! out = template.render( diff --git a/shpc/main/modules/templates/singularity.sh b/shpc/main/modules/templates/singularity.sh old mode 100644 new mode 100755 diff --git a/shpc/utils/fileio.py b/shpc/utils/fileio.py index 8c854efdf..0e8258d68 100644 --- a/shpc/utils/fileio.py +++ b/shpc/utils/fileio.py @@ -5,6 +5,7 @@ import hashlib import errno import os +import subprocess import re import shutil import tempfile @@ -114,12 +115,14 @@ def copyfile(source, destination, force=True): return destination -def write_file(filename, content, mode="w"): +def write_file(filename, content, mode="w", exec=False): """ Write content to a filename """ with open(filename, mode) as filey: filey.writelines(content) + if exec: + subprocess.call(["chmod", "+x", filename ]) return filename From 2c35b3ef3f829129e4246cb35ebc120bfd7e09b1 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Thu, 20 Jan 2022 11:06:29 +0800 Subject: [PATCH 12/38] wrapper scripts: templates for docker/podman --- shpc/main/modules/templates/docker.lua | 9 ++++++++- shpc/main/modules/templates/docker.sh | 3 +++ shpc/main/modules/templates/docker.tcl | 7 +++++++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100755 shpc/main/modules/templates/docker.sh diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua index f1ad7f41a..b2c1d28be 100644 --- a/shpc/main/modules/templates/docker.lua +++ b/shpc/main/modules/templates/docker.lua @@ -38,7 +38,7 @@ For each of the above, you can export: setenv ("PODMAN_OPTS", "") setenv ("PODMAN_COMMAND_OPTS", "") --- we probably don't need this +-- we probably do not need this local MODULEPATH="{{ module_dir }}" -- interactive shell to any container, plus exec for aliases @@ -56,6 +56,11 @@ set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) -- conflict with modules with the same name conflict(myModuleName(){% if aliases %}{% for alias in aliases %}{% if alias.name != name %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) +{% if wrapper_scripts %} +-- add path with "alias" wrapper scripts to PATH +{% if aliases %}prepend_path("PATH", "{{ wrapper_dir }}"){% endif %} + +{% else %} -- exec functions to provide "alias" to module commands {% if aliases %}{% for alias in aliases %} set_shell_function("{{ alias.name }}", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }} $@", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }}") @@ -67,6 +72,8 @@ if (myShellName() == "bash") then {% endfor %} end{% endif %} +{% endif %} + -- A customizable exec function set_shell_function("{|module_name|}-exec", execCmd .. " --entrypoint \"\" " .. containerPath .. " $@", execCmd .. " --entrypoint \"\" " .. containerPath) diff --git a/shpc/main/modules/templates/docker.sh b/shpc/main/modules/templates/docker.sh new file mode 100755 index 000000000..cb5403bf2 --- /dev/null +++ b/shpc/main/modules/templates/docker.sh @@ -0,0 +1,3 @@ +#!{{ wrapper_shell }} + +{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} --entrypoint {{ alias.entrypoint }} {{ image }} {{ alias.args }} {% if '/sh' in wrapper_shell or '/bash' in wrapper_shell %}$@{% elif '/csh' in wrapper_shell %}$*{% endif %} \ No newline at end of file diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl index 23c522936..8849306af 100644 --- a/shpc/main/modules/templates/docker.tcl +++ b/shpc/main/modules/templates/docker.tcl @@ -66,6 +66,11 @@ set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}" # set_shell_function takes bashStr and cshStr set-alias {|module_name|}-shell "${shellCmd}" +{% if wrapper_scripts %} +# add path with "alias" wrapper scripts to PATH +{% if aliases %}prepend-path PATH "{{ wrapper_dir }}"{% endif %} + +{% else %} # exec functions to provide "alias" to module commands {% if aliases %} if { [ module-info shell bash ] } { @@ -83,6 +88,8 @@ if { [ module-info shell bash ] } { } {% endif %} +{% endif %} + # A customizable exec function set-alias {|module_name|}-exec "${execCmd} --entrypoint \"\" ${containerPath}" From 98196732cbd47e235a94acbbd903cd0df8ac61ab Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Thu, 20 Jan 2022 14:37:47 +0800 Subject: [PATCH 13/38] wrapper scripts: bin directory under modules --- shpc/main/container/singularity.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/shpc/main/container/singularity.py b/shpc/main/container/singularity.py index 87af3de58..4f8883a54 100644 --- a/shpc/main/container/singularity.py +++ b/shpc/main/container/singularity.py @@ -169,7 +169,8 @@ def install( logger.warning("Singularity is not installed, skipping metadata.") # Option to create wrapper scripts for commands - wrapper_dir = os.path.join(container_dir, "bin") + module_dir = os.path.dirname(module_path) + wrapper_dir = os.path.join(module_dir, "bin") if self.settings.wrapper_scripts and aliases: shpc.utils.mkdirp([wrapper_dir]) for alias in aliases: @@ -179,7 +180,7 @@ def install( bindpaths=self.settings.bindpaths, container_sif=container_path, features=features, - module_dir=os.path.dirname(module_path), + module_dir=module_dir, envfile=self.settings.environment_file, wrapper_shell=self.settings.wrapper_shell, ) @@ -196,7 +197,7 @@ def install( url=url, features=features, version=version, - module_dir=os.path.dirname(module_path), + module_dir=module_dir, labels=labels, deffile=deffile, creation_date=datetime.now(), From 3309d4b381406e4fbc87efc957439406c32b967a Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Thu, 20 Jan 2022 14:46:48 +0800 Subject: [PATCH 14/38] wrapper scripts: docker/podman support --- shpc/main/container/docker.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/shpc/main/container/docker.py b/shpc/main/container/docker.py index b92145062..981d38204 100644 --- a/shpc/main/container/docker.py +++ b/shpc/main/container/docker.py @@ -189,6 +189,7 @@ def install( version=None, config_features=None, features=None, + wrapper_template=None, ): """Install a general container path to a module @@ -214,6 +215,26 @@ def install( # If there's a tag in the name, don't use it name = name.split(":", 1)[0] + # Option to create wrapper scripts for commands + module_dir = os.path.dirname(module_path) + wrapper_dir = os.path.join(module_dir, "bin") + if self.settings.wrapper_scripts and aliases: + shpc.utils.mkdirp([wrapper_dir]) + for alias in aliases: + wrapper_path = os.path.join(wrapper_dir, alias['name']) + out = wrapper_template.render( + alias=alias, + bindpaths=self.settings.bindpaths, + image=container_path, + module_dir=module_dir, + features=features, + envfile=self.settings.environment_file, + command=self.command, + tty=self.settings.enable_tty, + wrapper_shell=self.settings.wrapper_shell, + ) + shpc.utils.write_file(wrapper_path, out, exec=True) + # Make sure to render all values! out = template.render( podman_module=self.settings.podman_module, @@ -223,7 +244,7 @@ def install( else self.settings.docker_shell, image=container_path, description=description, - module_dir=os.path.dirname(module_path), + module_dir=module_dir, aliases=aliases, url=url, features=features, @@ -238,5 +259,7 @@ def install( envfile=self.settings.environment_file, command=self.command, tty=self.settings.enable_tty, + wrapper_scripts=self.settings.wrapper_scripts, + wrapper_dir=wrapper_dir, ) shpc.utils.write_file(module_path, out) From 6466461228b9c2d1d6afd62ace444c863ee3e98f Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Fri, 28 Jan 2022 11:36:33 +0800 Subject: [PATCH 15/38] reduced newlines in module templates --- shpc/main/modules/templates/docker.lua | 15 ++++----------- shpc/main/modules/templates/docker.tcl | 16 ++++------------ shpc/main/modules/templates/singularity.lua | 15 ++++----------- shpc/main/modules/templates/singularity.tcl | 16 ++++------------ 4 files changed, 16 insertions(+), 46 deletions(-) diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua index b2c1d28be..4583a8850 100644 --- a/shpc/main/modules/templates/docker.lua +++ b/shpc/main/modules/templates/docker.lua @@ -56,23 +56,16 @@ set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) -- conflict with modules with the same name conflict(myModuleName(){% if aliases %}{% for alias in aliases %}{% if alias.name != name %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) -{% if wrapper_scripts %} --- add path with "alias" wrapper scripts to PATH -{% if aliases %}prepend_path("PATH", "{{ wrapper_dir }}"){% endif %} - -{% else %} --- exec functions to provide "alias" to module commands -{% if aliases %}{% for alias in aliases %} -set_shell_function("{{ alias.name }}", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }} $@", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }}") +-- "aliases" to module commands +{% if wrapper_scripts %}{% if aliases %}prepend_path("PATH", "{{ wrapper_dir }}"){% endif %} +{% else %}{% if aliases %}{% for alias in aliases %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }} $@", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }}") {% endfor %}{% endif %} {% if aliases %} if (myShellName() == "bash") then {% for alias in aliases %}execute{cmd="export -f {{ alias.name }}", modeA={"load"}} {% endfor %} -end{% endif %} - -{% endif %} +end{% endif %}{% endif %} -- A customizable exec function set_shell_function("{|module_name|}-exec", execCmd .. " --entrypoint \"\" " .. containerPath .. " $@", execCmd .. " --entrypoint \"\" " .. containerPath) diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl index 8849306af..6c4dd481b 100644 --- a/shpc/main/modules/templates/docker.tcl +++ b/shpc/main/modules/templates/docker.tcl @@ -66,14 +66,9 @@ set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}" # set_shell_function takes bashStr and cshStr set-alias {|module_name|}-shell "${shellCmd}" -{% if wrapper_scripts %} -# add path with "alias" wrapper scripts to PATH -{% if aliases %}prepend-path PATH "{{ wrapper_dir }}"{% endif %} - -{% else %} -# exec functions to provide "alias" to module commands -{% if aliases %} -if { [ module-info shell bash ] } { +# "aliases" to module commands +{% if wrapper_scripts %}{% if aliases %}prepend-path PATH "{{ wrapper_dir }}"{% endif %} +{% else %}{% if aliases %}if { [ module-info shell bash ] } { if { [ module-info mode load ] } { {% for alias in aliases %} puts stdout "function {{ alias.name }}() { ${execCmd} {% if alias.docker_options %} {{ alias.docker_options | replace("$", "\$") }} {% endif %} --entrypoint {{ alias.entrypoint | replace("$", "\$") }} ${containerPath} {{ alias.args | replace("$", "\$") }} \$@; }; export -f {{ alias.name }};" {% endfor %} @@ -85,10 +80,7 @@ if { [ module-info shell bash ] } { } else { {% for alias in aliases %} set-alias {{ alias.name }} "${execCmd} {% if alias.docker_options %} {{ alias.docker_options | replace("$", "\$") }} {% endif %} --entrypoint {{ alias.entrypoint | replace("$", "\$") }} ${containerPath} {{ alias.args | replace("$", "\$") }}" {% endfor %} -} -{% endif %} - -{% endif %} +}{% endif %}{% endif %} # A customizable exec function set-alias {|module_name|}-exec "${execCmd} --entrypoint \"\" ${containerPath}" diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index 215c78055..bc4cbfbbf 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -60,23 +60,16 @@ set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) -- conflict with modules with the same name conflict(myModuleName(){% if aliases %}{% for alias in aliases %}{% if alias.name != name %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) -{% if wrapper_scripts %} --- add path with "alias" wrapper scripts to PATH -{% if aliases %}prepend_path("PATH", "{{ wrapper_dir }}"){% endif %} - -{% else %} --- exec functions to provide "alias" to module commands -{% if aliases %}{% for alias in aliases %} -set_shell_function("{{ alias.name }}", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }} $@", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }}") +-- "aliases" to module commands +{% if wrapper_scripts %}{% if aliases %}prepend_path("PATH", "{{ wrapper_dir }}"){% endif %} +{% else %}{% if aliases %}{% for alias in aliases %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }} $@", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }}") {% endfor %}{% endif %} {% if aliases %} if (myShellName() == "bash") then {% for alias in aliases %}execute{cmd="export -f {{ alias.name }}", modeA={"load"}} {% endfor %} -end{% endif %} - -{% endif %} +end{% endif %}{% endif %} -- A customizable exec function set_shell_function("{|module_name|}-exec", execCmd .. containerPath .. " $@", execCmd .. containerPath) diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl index ee5c027b0..80bd2e1d6 100644 --- a/shpc/main/modules/templates/singularity.tcl +++ b/shpc/main/modules/templates/singularity.tcl @@ -74,14 +74,9 @@ set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_ # set_shell_function takes bashStr and cshStr set-alias {|module_name|}-shell "${shellCmd}" -{% if wrapper_scripts %} -# add path with "alias" wrapper scripts to PATH -{% if aliases %}prepend-path PATH "{{ wrapper_dir }}"{% endif %} - -{% else %} -# exec functions to provide "alias" to module commands -{% if aliases %} -if { [ module-info shell bash ] } { +# "aliases" to module commands +{% if wrapper_scripts %}{% if aliases %}prepend-path PATH "{{ wrapper_dir }}"{% endif %} +{% else %}{% if aliases %}if { [ module-info shell bash ] } { if { [ module-info mode load ] } { {% for alias in aliases %} puts stdout "function {{ alias.name }}() { ${execCmd} {% if alias.singularity_options %} {{ alias.singularity_options | replace("$", "\$") }} {% endif %} ${containerPath} {{ alias.command | replace("$", "\$") }} \$@; }; export -f {{ alias.name }};" {% endfor %} @@ -93,10 +88,7 @@ if { [ module-info shell bash ] } { } else { {% for alias in aliases %} set-alias {{ alias.name }} "${execCmd} {% if alias.singularity_options %} {{ alias.singularity_options | replace("$", "\$") }} {% endif %} ${containerPath} {{ alias.command | replace("$", "\$") }}" {% endfor %} -} -{% endif %} - -{% endif %} +}{% endif %}{% endif %} # A customizable exec function set-alias {|module_name|}-exec "${execCmd} ${containerPath}" From 028a1d6c1e382e36f5b9180f56d4ef6e5715eb10 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Fri, 28 Jan 2022 11:42:02 +0800 Subject: [PATCH 16/38] all shell settings nearby in settings.yml --- shpc/settings.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shpc/settings.yml b/shpc/settings.yml index bdd898509..df3521d62 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -50,12 +50,12 @@ docker_shell: /bin/sh # shell for test.sh file test_shell: /bin/bash -# for container aliases, use wrapper shellscripts instead of shell aliases, defaults to false -wrapper_scripts: false - # shell for wrapper shellscripts wrapper_shell: /bin/bash +# for container aliases, use wrapper shellscripts instead of shell aliases, defaults to false +wrapper_scripts: false + # the module namespace you want to install from. E.g., if you have ghcr.io/autamus/clingo # and you set the namespace to ghcr.io/autamus, you can just do: shpc install clingo. namespace: From 0d433265cba7d6700559105391ec15a64737e44d Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Fri, 28 Jan 2022 14:48:13 +0800 Subject: [PATCH 17/38] utils/fileio: make executable using stat rather than subprocess methods --- shpc/utils/fileio.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/shpc/utils/fileio.py b/shpc/utils/fileio.py index 0e8258d68..b8c5bb30f 100644 --- a/shpc/utils/fileio.py +++ b/shpc/utils/fileio.py @@ -5,7 +5,7 @@ import hashlib import errno import os -import subprocess +import stat import re import shutil import tempfile @@ -122,7 +122,8 @@ def write_file(filename, content, mode="w", exec=False): with open(filename, mode) as filey: filey.writelines(content) if exec: - subprocess.call(["chmod", "+x", filename ]) + st = os.stat(filename) + os.chmod(filename, st.st_mode | stat.S_IEXEC) return filename From 8646f315ed1b7c0c1d618bb1e18afce4951c5d7d Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Fri, 4 Feb 2022 17:05:40 +0800 Subject: [PATCH 18/38] wrapper script templates: protecting argument passing --- shpc/main/modules/templates/singularity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shpc/main/modules/templates/singularity.sh b/shpc/main/modules/templates/singularity.sh index 2766a917d..9b12aa58d 100755 --- a/shpc/main/modules/templates/singularity.sh +++ b/shpc/main/modules/templates/singularity.sh @@ -1,3 +1,3 @@ #!{{ wrapper_shell }} -singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} {% if alias.singularity_options %} {{ alias.singularity_options }} {% endif %} {{ container_sif }} {{ alias.command }} {% if '/sh' in wrapper_shell or '/bash' in wrapper_shell %}$@{% elif '/csh' in wrapper_shell %}$*{% endif %} \ No newline at end of file +singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} {% if alias.singularity_options %} {{ alias.singularity_options }} {% endif %} {{ container_sif }} {{ alias.command }} {% if '/sh' in wrapper_shell or '/bash' in wrapper_shell %}"$@"{% elif '/csh' in wrapper_shell %}$argv:q{% endif %} \ No newline at end of file From 4587b2b6cda2d6165536e8130bce15febaa72c4f Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Mon, 14 Feb 2022 10:37:32 +0800 Subject: [PATCH 19/38] wrapper script templates: protecting argument passing for docker, too --- shpc/main/modules/templates/docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shpc/main/modules/templates/docker.sh b/shpc/main/modules/templates/docker.sh index cb5403bf2..4e4f61bee 100755 --- a/shpc/main/modules/templates/docker.sh +++ b/shpc/main/modules/templates/docker.sh @@ -1,3 +1,3 @@ #!{{ wrapper_shell }} -{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} --entrypoint {{ alias.entrypoint }} {{ image }} {{ alias.args }} {% if '/sh' in wrapper_shell or '/bash' in wrapper_shell %}$@{% elif '/csh' in wrapper_shell %}$*{% endif %} \ No newline at end of file +{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} --entrypoint {{ alias.entrypoint }} {{ image }} {{ alias.args }} {% if '/sh' in wrapper_shell or '/bash' in wrapper_shell %}"$@"{% elif '/csh' in wrapper_shell %}$argv:q{% endif %} From 1f03dbdca91add18994a544bc720087374a70b25 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Tue, 15 Feb 2022 21:57:42 +0800 Subject: [PATCH 20/38] container: defined wrapper_subdir attribute --- shpc/main/container/base.py | 3 +++ shpc/main/container/docker.py | 2 +- shpc/main/container/singularity.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/shpc/main/container/base.py b/shpc/main/container/base.py index 92e460e4a..52752150e 100644 --- a/shpc/main/container/base.py +++ b/shpc/main/container/base.py @@ -46,6 +46,9 @@ class ContainerTechnology: # The module technology adds extensions here modulefile = "module" + # Wrapper scripts are stored in this subdirectory of the module directory + wrapper_subdir = "bin" + # By default, no extra features features = {} diff --git a/shpc/main/container/docker.py b/shpc/main/container/docker.py index 8c9f82b9d..1d5cd4152 100644 --- a/shpc/main/container/docker.py +++ b/shpc/main/container/docker.py @@ -217,7 +217,7 @@ def install( # Option to create wrapper scripts for commands module_dir = os.path.dirname(module_path) - wrapper_dir = os.path.join(module_dir, "bin") + wrapper_dir = os.path.join(module_dir, self.wrapper_subdir) if self.settings.wrapper_scripts and aliases: shpc.utils.mkdirp([wrapper_dir]) for alias in aliases: diff --git a/shpc/main/container/singularity.py b/shpc/main/container/singularity.py index 295aa1508..53f3c6fdf 100644 --- a/shpc/main/container/singularity.py +++ b/shpc/main/container/singularity.py @@ -170,7 +170,7 @@ def install( # Option to create wrapper scripts for commands module_dir = os.path.dirname(module_path) - wrapper_dir = os.path.join(module_dir, "bin") + wrapper_dir = os.path.join(module_dir, self.wrapper_subdir) if self.settings.wrapper_scripts and aliases: shpc.utils.mkdirp([wrapper_dir]) for alias in aliases: From 4cf7aabea5a33ad94a91f082f3fc01c16916384f Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Tue, 15 Feb 2022 21:58:25 +0800 Subject: [PATCH 21/38] modules: defined wrappertemplatefile property --- shpc/main/modules/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/shpc/main/modules/__init__.py b/shpc/main/modules/__init__.py index 066f8cc74..4c71ab64b 100644 --- a/shpc/main/modules/__init__.py +++ b/shpc/main/modules/__init__.py @@ -79,6 +79,10 @@ def modulefile(self): def templatefile(self): return "%s.%s" % (self.container.templatefile, self.module_extension) + @property + def wrappertemplatefile(self): + return "%s.%s" % (self.container.templatefile, "sh") + def uninstall(self, name, force=False): """ Given a unique resource identifier, uninstall a module @@ -147,7 +151,7 @@ def add(self, sif, module_name, **kwargs): module_name = self.add_namespace(module_name) template = self._load_template(self.templatefile) modulefile = os.path.join(self.settings.module_base, module_name.replace(":", os.sep), self.modulefile) - wrapper_template = self._load_template('.'.join([self.container.templatefile,'sh'])) + wrapper_template = self._load_template(self.wrappertemplatefile) self.container.add(sif, module_name, modulefile, template, wrapper_template, **kwargs) def get(self, module_name, env_file=False): @@ -326,7 +330,7 @@ def install(self, name, tag=None, **kwargs): # Get the template based on the module and container type template = self._load_template(self.templatefile) module_path = os.path.join(module_dir, self.modulefile) - wrapper_template = self._load_template('.'.join([self.container.templatefile,'sh'])) + wrapper_template = self._load_template(self.wrappertemplatefile) # If the module has a version, overrides version version = tag.name From be75776001ad312d7fa2f1b0a36775e8d7fec336 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Tue, 15 Feb 2022 22:11:53 +0800 Subject: [PATCH 22/38] wrapper scripts: relative PATH in modulefile templates --- shpc/main/container/docker.py | 2 +- shpc/main/container/singularity.py | 2 +- shpc/main/modules/templates/docker.lua | 2 +- shpc/main/modules/templates/docker.tcl | 2 +- shpc/main/modules/templates/singularity.lua | 2 +- shpc/main/modules/templates/singularity.tcl | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shpc/main/container/docker.py b/shpc/main/container/docker.py index 1d5cd4152..b86f5905d 100644 --- a/shpc/main/container/docker.py +++ b/shpc/main/container/docker.py @@ -260,6 +260,6 @@ def install( command=self.command, tty=self.settings.enable_tty, wrapper_scripts=self.settings.wrapper_scripts, - wrapper_dir=wrapper_dir, + wrapper_subdir=self.wrapper_subdir, ) shpc.utils.write_file(module_path, out) diff --git a/shpc/main/container/singularity.py b/shpc/main/container/singularity.py index 53f3c6fdf..f809b97eb 100644 --- a/shpc/main/container/singularity.py +++ b/shpc/main/container/singularity.py @@ -207,7 +207,7 @@ def install( repository=parsed_name.repository, envfile=self.settings.environment_file, wrapper_scripts=self.settings.wrapper_scripts, - wrapper_dir=wrapper_dir, + wrapper_subdir=self.wrapper_subdir, ) shpc.utils.write_file(module_path, out) diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua index 4171ed9b8..2c559530e 100644 --- a/shpc/main/modules/templates/docker.lua +++ b/shpc/main/modules/templates/docker.lua @@ -59,7 +59,7 @@ set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) conflict("{{ tool }}"{% if name != tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) -- "aliases" to module commands -{% if wrapper_scripts %}{% if aliases %}prepend_path("PATH", "{{ wrapper_dir }}"){% endif %} +{% if wrapper_scripts %}{% if aliases %}prepend_path("PATH", pathJoin(myFileName():match("(.*[/])") or ".", "{{ wrapper_subdir }}")){% endif %} {% else %}{% if aliases %}{% for alias in aliases %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }} \"$@\"", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }}") {% endfor %}{% endif %} diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl index e68334d6b..9d71f8c0e 100644 --- a/shpc/main/modules/templates/docker.tcl +++ b/shpc/main/modules/templates/docker.tcl @@ -74,7 +74,7 @@ set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}" set-alias {|module_name|}-shell "${shellCmd}" # "aliases" to module commands -{% if wrapper_scripts %}{% if aliases %}prepend-path PATH "{{ wrapper_dir }}"{% endif %} +{% if wrapper_scripts %}{% if aliases %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/{{ wrapper_subdir }}"{% endif %} {% else %}{% if aliases %}if { [ module-info shell bash ] } { if { [ module-info mode load ] } { {% for alias in aliases %} puts stdout "function {{ alias.name }}() { ${execCmd} {% if alias.docker_options %} {{ alias.docker_options | replace("$", "\$") }} {% endif %} --entrypoint {{ alias.entrypoint | replace("$", "\$") }} ${containerPath} {{ alias.args | replace("$", "\$") }} \"\$@\"; }; export -f {{ alias.name }};" diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index 4127d9ed9..d3674ce33 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -62,7 +62,7 @@ set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) conflict("{{ tool }}"{% if name != tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) -- "aliases" to module commands -{% if wrapper_scripts %}{% if aliases %}prepend_path("PATH", "{{ wrapper_dir }}"){% endif %} +{% if wrapper_scripts %}{% if aliases %}prepend_path("PATH",pathJoin(myFileName():match("(.*[/])") or ".", "{{ wrapper_subdir }}")){% endif %} {% else %}{% if aliases %}{% for alias in aliases %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }} \"$@\"", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }}") {% endfor %}{% endif %} diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl index 127452ac3..a69868212 100644 --- a/shpc/main/modules/templates/singularity.tcl +++ b/shpc/main/modules/templates/singularity.tcl @@ -79,7 +79,7 @@ set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_ set-alias {|module_name|}-shell "${shellCmd}" # "aliases" to module commands -{% if wrapper_scripts %}{% if aliases %}prepend-path PATH "{{ wrapper_dir }}"{% endif %} +{% if wrapper_scripts %}{% if aliases %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/{{ wrapper_subdir }}"{% endif %} {% else %}{% if aliases %}if { [ module-info shell bash ] } { if { [ module-info mode load ] } { {% for alias in aliases %} puts stdout "function {{ alias.name }}() { ${execCmd} {% if alias.singularity_options %} {{ alias.singularity_options | replace("$", "\$") }} {% endif %} ${containerPath} {{ alias.command | replace("$", "\$") }} \"\$@\"; }; export -f {{ alias.name }};" From cf5b0c37783203463985a371a5b06c1d29befb16 Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Wed, 16 Feb 2022 10:34:23 +0800 Subject: [PATCH 23/38] wrapper_scripts: new function _generate_wrapper_scripts in container/base.py --- shpc/main/container/base.py | 23 +++++++++++++++++++++++ shpc/main/container/docker.py | 25 +++++++++---------------- shpc/main/container/singularity.py | 21 +++++++-------------- 3 files changed, 39 insertions(+), 30 deletions(-) diff --git a/shpc/main/container/base.py b/shpc/main/container/base.py index 52752150e..ad09a4494 100644 --- a/shpc/main/container/base.py +++ b/shpc/main/container/base.py @@ -178,5 +178,28 @@ def get_features(self, config_features, settings_features, extra=None): return features + def _generate_wrapper_scripts(self, wrapper_template, aliases, module_dir, features, command=None, container_sif=None, image=None, tty=None): + """ + Generate wrapper scripts for commands (when wrapper_scripts setting enabled) + """ + wrapper_dir = os.path.join(module_dir, self.wrapper_subdir) + shpc.utils.mkdirp([wrapper_dir]) + for alias in aliases: + wrapper_path = os.path.join(wrapper_dir, alias['name']) + out = wrapper_template.render( + alias=alias, + bindpaths=self.settings.bindpaths, + command=command, + container_sif=container_sif, + envfile=self.settings.environment_file, + features=features, + image=image, + module_dir=module_dir, + tty=tty, + wrapper_shell=self.settings.wrapper_shell, + ) + shpc.utils.write_file(wrapper_path, out, exec=True) + return + def __str__(self): return str(self.__class__.__name__) diff --git a/shpc/main/container/docker.py b/shpc/main/container/docker.py index b86f5905d..aaf023760 100644 --- a/shpc/main/container/docker.py +++ b/shpc/main/container/docker.py @@ -217,23 +217,16 @@ def install( # Option to create wrapper scripts for commands module_dir = os.path.dirname(module_path) - wrapper_dir = os.path.join(module_dir, self.wrapper_subdir) if self.settings.wrapper_scripts and aliases: - shpc.utils.mkdirp([wrapper_dir]) - for alias in aliases: - wrapper_path = os.path.join(wrapper_dir, alias['name']) - out = wrapper_template.render( - alias=alias, - bindpaths=self.settings.bindpaths, - image=container_path, - module_dir=module_dir, - features=features, - envfile=self.settings.environment_file, - command=self.command, - tty=self.settings.enable_tty, - wrapper_shell=self.settings.wrapper_shell, - ) - shpc.utils.write_file(wrapper_path, out, exec=True) + self._generate_wrapper_scripts( + wrapper_template, + aliases, + module_dir, + features, + command=self.command, + image=container_path, + tty=self.settings.enable_tty, + ) # Make sure to render all values! out = template.render( diff --git a/shpc/main/container/singularity.py b/shpc/main/container/singularity.py index f809b97eb..e71a54cc9 100644 --- a/shpc/main/container/singularity.py +++ b/shpc/main/container/singularity.py @@ -170,21 +170,14 @@ def install( # Option to create wrapper scripts for commands module_dir = os.path.dirname(module_path) - wrapper_dir = os.path.join(module_dir, self.wrapper_subdir) if self.settings.wrapper_scripts and aliases: - shpc.utils.mkdirp([wrapper_dir]) - for alias in aliases: - wrapper_path = os.path.join(wrapper_dir, alias['name']) - out = wrapper_template.render( - alias=alias, - bindpaths=self.settings.bindpaths, - container_sif=container_path, - features=features, - module_dir=module_dir, - envfile=self.settings.environment_file, - wrapper_shell=self.settings.wrapper_shell, - ) - shpc.utils.write_file(wrapper_path, out, exec=True) + self._generate_wrapper_scripts( + wrapper_template, + aliases, + module_dir, + features, + container_sif=container_path, + ) # Make sure to render all values! out = template.render( From 28a459460c266c0179e4eb4ce5f70dbaead7c18c Mon Sep 17 00:00:00 2001 From: vsoch Date: Wed, 16 Feb 2022 02:52:40 -0700 Subject: [PATCH 24/38] first shot at adding wrapper scripts this will support two kinds of wrapper scripts - global that live in the settings.yaml, and container.yaml specific that go alongside that file. We basically will then generate aliases for any executable names that were not created as wrappers. we will want to follow this up with a command to list wrappers, and also include descriptions for the interested user, and a flag to install to select on the fly Signed-off-by: vsoch --- CHANGELOG.md | 1 + docs/getting_started/developer-guide.rst | 154 +++++++++++++++++++- docs/getting_started/user-guide.rst | 59 ++++++++ registry/vanessa/salad/container.yaml | 8 + registry/vanessa/salad/docker_fork.sh | 7 + registry/vanessa/salad/singularity_fork.sh | 7 + shpc/main/container/__init__.py | 4 +- shpc/main/container/base.py | 28 +--- shpc/main/container/docker.py | 47 +++--- shpc/main/container/singularity.py | 38 +++-- shpc/main/modules/__init__.py | 10 +- shpc/main/modules/templates/docker.lua | 30 ++-- shpc/main/modules/templates/docker.sh | 3 - shpc/main/modules/templates/docker.tcl | 36 ++--- shpc/main/modules/templates/singularity.lua | 30 ++-- shpc/main/modules/templates/singularity.sh | 3 - shpc/main/modules/templates/singularity.tcl | 40 ++--- shpc/main/schemas.py | 26 +++- shpc/main/wrappers/__init__.py | 62 ++++++++ shpc/main/wrappers/base.py | 136 +++++++++++++++++ shpc/main/wrappers/docker.sh | 3 + shpc/main/wrappers/scripts.py | 24 +++ shpc/main/wrappers/singularity.sh | 3 + shpc/settings.yml | 22 ++- shpc/version.py | 2 +- 25 files changed, 623 insertions(+), 160 deletions(-) create mode 100755 registry/vanessa/salad/docker_fork.sh create mode 100755 registry/vanessa/salad/singularity_fork.sh delete mode 100755 shpc/main/modules/templates/docker.sh delete mode 100755 shpc/main/modules/templates/singularity.sh create mode 100644 shpc/main/wrappers/__init__.py create mode 100644 shpc/main/wrappers/base.py create mode 100755 shpc/main/wrappers/docker.sh create mode 100644 shpc/main/wrappers/scripts.py create mode 100755 shpc/main/wrappers/singularity.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 538f1dc7e..806163914 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are: The versions coincide with releases on pip. Only major versions will be released as tags on Github. ## [0.0.x](https://github.com/singularityhub/singularity-hpc/tree/main) (0.0.x) + - Adding support for wrapper scripts for global and container.yaml (0.0.45) - Lua and tcl module file bug fixes for shell functions and aliases (0.0.44) - restoring -B for Singularity bindpaths over using envar - default containers directory should be separate from modules (0.0.43) diff --git a/docs/getting_started/developer-guide.rst b/docs/getting_started/developer-guide.rst index aa5280dc0..b67131dbe 100644 --- a/docs/getting_started/developer-guide.rst +++ b/docs/getting_started/developer-guide.rst @@ -198,6 +198,151 @@ For each of the above, depending on the prefix of options that you choose, it wi This means that if you design a new registry recipe, you should consider how to run it for both kinds of technology. Also note that ``docker_options`` are those that will also be used for Podman. +Wrapper Script +-------------- + +Singularity HPC allows exposure of two kinds of wrapper scripts: + +1. A global level wrapper intended to replace aliases. E.g., if an alias "samtools" is typically a direct container call, enabling a wrapper will generate an executable script "samtools" in a bin associated with the container, added to the path, to call instead. This is desired for workflow tools that want to run scripts. This global script is defined in settings.yml and described in the user guide. +2. A container level wrapper that is specific to a container, described here. + +For container specific scripts, you can add sections to a ``container.yaml`` to specify the script (and container type) +and the scripts must be provided alongside the container.yaml to install. + +.. code-block:: yaml + + docker_scripts: + fork: docker_fork.sh + singularity_scripts: + fork: singularity_fork.sh + +The above says "given generation of a docker or podman container, write a script named "fork" that uses "docker_fork.sh" as a template" +and the same for Singularity. And then I (the developer) would provide the custom scripts alongside container.yaml: + +.. code-block:: console + + registry/vanessa/salad/ + ├── container.yaml + ├── docker_fork.sh + └── singularity_fork.sh + +You can look at ``registry/vanessa/salad`` for an example that includes Singularity +and Docker wrapper scripts. For example, when generating for a singularity container with +the global wrapped scripts enabled, we get one wrapper script for the alias "salad" and one for +the custom container script "fork": + +.. code-block:: console + + $ tree containers/vanessa/salad/ + containers/vanessa/salad/ + └── latest + ├── bin + │   ├── fork + │   └── salad + └── vanessa-salad-latest-sha256:e8302da47e3200915c1d3a9406d9446f04da7244e4995b7135afd2b79d4f63db.sif + +If we disable all wrapper scripts, the bin directory would not exist. If we set the default wrapper +scripts for singularity and docker in settings.yml and left enable to true, we would only see "fork." + +How to write an alias wrapper script +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +First, decide if you want a global script (to replace or wrap aliases) OR a custom container script. For an alias derived script, you should: + +1. Write the new script file into shpc/main/wrappers. +2. Add an entry to shpc/main/wrappers/scripts referencing the script. + +For global scripts, the user can select to use it in their settings.yaml. +We will eventually write a command to list global wrappers available, so if you add a new one future users will know +about it. For alias wrapper scripts, the following variables are passed for rendering: + + +.. list-table:: Title + :widths: 15 15 40 30 + :header-rows: 1 + + * - Name + - Type + - Description + - Example + * - alias + - dictionary + - The entire alias in question, including subfields name, command, singularity_options or docker_options, and args + - ``{{ alias.name }}`` + * - settings + - dictionary + - Everything referenced in the user settings + - ``{{ settings.wrapper_shell }}`` + * - container + - dictionary + - The container technology + - ``{{ container.command }}`` renders to docker, singularity, or podman + * - config + - dictionary + - The entire container config (container.yaml) structured the same + - ``{{ config.docker }}`` + * - image + - string + - The name of the container binary (SIF) or unique resource identifier + - ``{{ image }}`` + * - module_dir + - string + - The name of the module directory + - ``{{ module_dir }}`` + * - features + - dictionary + - A dictionary of parsed features + - ``{{ features.gpu }}`` + + + +How to write an container wrapper script +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you want to write a custom container.yaml script: + +1. Add either (or both) of singularity_scripts/docker_scripts in the container.yaml, including an alias command and an associated script. +2. Write the script with the associated name into that folder. + +The following variables are passed for rendering. + +.. list-table:: Title + :widths: 15 15 40 30 + :header-rows: 1 + + * - Name + - Type + - Description + - Example + * - alias + - string + - The alias name defined under singularity_scripts or docker_scripts + - ``{{ alias }}`` + * - settings + - dictionary + - Everything referenced in the user settings + - ``{{ settings.wrapper_shell }}`` + * - container + - dictionary + - The container technology + - ``{{ container.command }}`` renders to docker, singularity, or podman + * - config + - dictionary + - The entire container config (container.yaml) structured the same + - ``{{ config.docker }}`` + * - image + - string + - The name of the container binary (SIF) or unique resource identifier + - ``{{ image }}`` + * - module_dir + - string + - The name of the module directory + - ``{{ module_dir }}`` + * - features + - dictionary + - A dictionary of parsed features + - ``{{ features.gpu }}`` + Environment Variables --------------------- @@ -337,7 +482,7 @@ Fields include: - A list of patterns to use for adding new tags. If not defined, all are added - false * - aliases - - Named entrypoints for container (dict) + - Named entrypoints for container (dict) as described above - false * - url - Documentation or other url for the container uri @@ -351,7 +496,12 @@ Fields include: * - features - Optional key, value paired set of features to enable for the container. Currently allowed keys: *gpu* *home* and *x11*. - varies - + * - singularity_scripts + - key value pairs of wrapper names (e.g., executable called by user) and local container script for Singularity + - false + * - docker_scripts + - key value pairs of wrapper names (e.g., executable called by user) and local container script for Docker or Singularity + - false A complete table of features is shown here. The diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst index 3e2f65d56..12e0021d8 100644 --- a/docs/getting_started/user-guide.rst +++ b/docs/getting_started/user-guide.rst @@ -446,6 +446,65 @@ If you would like support for a different container technology that has not been mentioned, please also `open an issue `_ and provide description and links to what you have in mind. +Wrapper Scripts +--------------- + +Singularity HPC allows for global definition of wrapper scripts, meaning that instead of writing a module alias to run a container for some given alias, +we generate a wrapper script of the same name instead. Since the settings.yml is global, all wrapper scripts defined here are specific to replacing aliases. +Container-specific scripts you'll want to include in the container.yaml, described in the developer docs. Let's take a look at the settings: + +# Wrapper scripts to use for aliases (replaces them) +# Put a fullpath here for a custom script, otherwise we look in shpc/main/wrappers +# If you want a custom wrapper script for a one-off container, define it there with the script +# stored alongside the container.yaml. + + +.. code-block:: yaml + + wrapper_scripts: + + # Enable wrapper scripts, period. If enabled, generate scripts for aliases instead of commands + # if enabled, we also allow container-specific wrapper scripts. + enabled: false + + # use for docker aliases + docker: docker.sh + + # use for podman aliases + podman: docker.sh + + # use for singularity aliases + singularity: singularity.sh + +Since different container technologies might expose different environment variables (e.g., ``SINGULARITY_OPTS`` vs ``PODMAN_OPTS``) +they are organized above based on the container technology. If you want to customize the wrapper script, simply replace the relative paths +above (e.g., ``singularity.sh`` with an absolute path to a file that will be used instead. For global alias scripts such as these, +Singularity HPC will look for: + +1. An absolute path first, if found is used first. +2. Then a script name in the shpc/main/wrappers directory + +For container specific scripts, you can add sections to a ``container.yaml`` to specify the script (and container type) +and the scripts must be provided alongside the container.yaml to install. + +.. code-block:: yaml + + docker_scripts: + fork: docker_fork.sh + singularity_scripts: + fork: singularity_fork.sh + +The above says "given generation of a docker or podman container, write a script named "fork" that uses "docker_fork.sh" as a template" +and the same for Singularity. And then I (the developer) would provide the custom scripts alongside container.yaml: + +.. code-block:: console + + registry/vanessa/salad/ + ├── container.yaml + ├── docker_fork.sh + └── singularity_fork.sh + + .. _getting_started-commands: diff --git a/registry/vanessa/salad/container.yaml b/registry/vanessa/salad/container.yaml index 64b582ea5..9fba0b336 100644 --- a/registry/vanessa/salad/container.yaml +++ b/registry/vanessa/salad/container.yaml @@ -8,5 +8,13 @@ tags: latest: sha256:e8302da47e3200915c1d3a9406d9446f04da7244e4995b7135afd2b79d4f63db aliases: salad: /code/salad + +# An example of a custom wrapper script, stored here, will generate "fork" alias +# for each of a docker and singularity container +docker_scripts: + fork: docker_fork.sh +singularity_scripts: + fork: singularity_fork.sh + env: maintainer: vsoch diff --git a/registry/vanessa/salad/docker_fork.sh b/registry/vanessa/salad/docker_fork.sh new file mode 100755 index 000000000..ebee68583 --- /dev/null +++ b/registry/vanessa/salad/docker_fork.sh @@ -0,0 +1,7 @@ +#!{{ settings.wrapper_shell }} + +# This is an example of a custom wrapper script for a docker container +# Note that we are showing an example of all the default args provided to the script! +# You don't need to use any of this syntax + +{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} --entrypoint /code/salad {{ image }} fork {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} diff --git a/registry/vanessa/salad/singularity_fork.sh b/registry/vanessa/salad/singularity_fork.sh new file mode 100755 index 000000000..f6821f26a --- /dev/null +++ b/registry/vanessa/salad/singularity_fork.sh @@ -0,0 +1,7 @@ +#!{{ settings.wrapper_shell }} + +# This is an example of a singularity wrapper script, under singularity_scripts +# Note that we are showing an example of all the default args provided to the script! +# You don't need to use any of this syntax + +singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environent_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {{ image }} /code/salad fork {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} diff --git a/shpc/main/container/__init__.py b/shpc/main/container/__init__.py index f3c1e1181..87f33cfb9 100644 --- a/shpc/main/container/__init__.py +++ b/shpc/main/container/__init__.py @@ -63,7 +63,9 @@ def __repr__(self): class ContainerConfig: - """A ContainerConfig wraps a container.yaml file, intended for install.""" + """ + A ContainerConfig wraps a container.yaml file, intended for install. + """ def __init__(self, package_file): """Load a package file for a container.""" diff --git a/shpc/main/container/base.py b/shpc/main/container/base.py index ad09a4494..fcb571314 100644 --- a/shpc/main/container/base.py +++ b/shpc/main/container/base.py @@ -46,9 +46,6 @@ class ContainerTechnology: # The module technology adds extensions here modulefile = "module" - # Wrapper scripts are stored in this subdirectory of the module directory - wrapper_subdir = "bin" - # By default, no extra features features = {} @@ -60,7 +57,7 @@ def __init__(self): self.settings = SettingsBase() - def add(self, sif, module_name, modulefile, template, wrapper_template, **kwargs): + def add(self, sif, module_name, modulefile, template, **kwargs): """ Manually add a registry container. """ @@ -178,28 +175,5 @@ def get_features(self, config_features, settings_features, extra=None): return features - def _generate_wrapper_scripts(self, wrapper_template, aliases, module_dir, features, command=None, container_sif=None, image=None, tty=None): - """ - Generate wrapper scripts for commands (when wrapper_scripts setting enabled) - """ - wrapper_dir = os.path.join(module_dir, self.wrapper_subdir) - shpc.utils.mkdirp([wrapper_dir]) - for alias in aliases: - wrapper_path = os.path.join(wrapper_dir, alias['name']) - out = wrapper_template.render( - alias=alias, - bindpaths=self.settings.bindpaths, - command=command, - container_sif=container_sif, - envfile=self.settings.environment_file, - features=features, - image=image, - module_dir=module_dir, - tty=tty, - wrapper_shell=self.settings.wrapper_shell, - ) - shpc.utils.write_file(wrapper_path, out, exec=True) - return - def __str__(self): return str(self.__class__.__name__) diff --git a/shpc/main/container/docker.py b/shpc/main/container/docker.py index aaf023760..d32a177c1 100644 --- a/shpc/main/container/docker.py +++ b/shpc/main/container/docker.py @@ -6,6 +6,7 @@ from shpc.logger import logger from .base import ContainerTechnology import shpc.main.templates +import shpc.main.wrappers import shpc.utils from datetime import datetime @@ -189,7 +190,7 @@ def install( version=None, config_features=None, features=None, - wrapper_template=None, + config=None, ): """Install a general container path to a module @@ -217,24 +218,30 @@ def install( # Option to create wrapper scripts for commands module_dir = os.path.dirname(module_path) - if self.settings.wrapper_scripts and aliases: - self._generate_wrapper_scripts( - wrapper_template, - aliases, - module_dir, - features, - command=self.command, - image=container_path, - tty=self.settings.enable_tty, + wrapper_scripts = None + + # Wrapper scripts can be global (for aliases) or container specific + if self.settings.wrapper_scripts["enabled"] is True: + wrapper_scripts = shpc.main.wrappers.generate( + aliases=aliases, + module_dir=module_dir, + features=features, + container=self, + image=container_path, + config=config, ) - # Make sure to render all values! - out = template.render( - podman_module=self.settings.podman_module, - bindpaths=self.settings.bindpaths, - shell=self.settings.podman_shell + # What shell to use? + shell = ( + self.settings.podman_shell if self.command == "podman" else self.settings.docker_shell, + ) + + # Make sure to render all values! + out = template.render( + settings=self.settings, + shell=shell, image=container_path, description=description, module_dir=module_dir, @@ -243,16 +250,10 @@ def install( features=features, version=version, labels=labels, - prefix=self.settings.module_exc_prefix, creation_date=datetime.now(), name=name, - tool=parsed_name.tool, - registry=parsed_name.registry, - repository=parsed_name.repository, - envfile=self.settings.environment_file, + parsed_name=parsed_name, command=self.command, - tty=self.settings.enable_tty, - wrapper_scripts=self.settings.wrapper_scripts, - wrapper_subdir=self.wrapper_subdir, + wrapper_scripts=wrapper_scripts, ) shpc.utils.write_file(module_path, out) diff --git a/shpc/main/container/singularity.py b/shpc/main/container/singularity.py index e71a54cc9..b7d8cc02a 100644 --- a/shpc/main/container/singularity.py +++ b/shpc/main/container/singularity.py @@ -23,6 +23,7 @@ class SingularityContainer(ContainerTechnology): # The module technology adds extensions here templatefile = "singularity" + command = "singularity" # Singularity container features features = { @@ -71,7 +72,7 @@ def get(self, module_name, env_file=False): logger.exit("Found more than one sif in module folder.") return sif[0] - def add(self, sif, module_name, modulefile, template, wrapper_template, **kwargs): + def add(self, sif, module_name, modulefile, template, **kwargs): """ Manually add a registry container. """ @@ -112,7 +113,6 @@ def add(self, sif, module_name, modulefile, template, wrapper_template, **kwargs template, parsed_name=parsed_name, features=kwargs.get("features"), - wrapper_template=wrapper_template, ) self.add_environment(module_dir, {}, self.settings.environment_file) logger.info("Module %s was created." % (module_name)) @@ -131,7 +131,7 @@ def install( config_features=None, features=None, version=None, - wrapper_template=None, + config=None, ): """Install a general container path to a module @@ -170,20 +170,22 @@ def install( # Option to create wrapper scripts for commands module_dir = os.path.dirname(module_path) - if self.settings.wrapper_scripts and aliases: - self._generate_wrapper_scripts( - wrapper_template, - aliases, - module_dir, - features, - container_sif=container_path, + + # Wrapper scripts can be global (for aliases) or container specific + wrapper_scripts = None + if self.settings.wrapper_scripts["enabled"] is True: + wrapper_scripts = shpc.main.wrappers.generate( + aliases=aliases, + module_dir=module_dir, + features=features, + container=self, + image=container_path, + config=config, ) # Make sure to render all values! out = template.render( - singularity_module=self.settings.singularity_module, - singularity_shell=self.settings.singularity_shell, - bindpaths=self.settings.bindpaths, + settings=self.settings, container_sif=container_path, description=description, aliases=aliases, @@ -195,12 +197,8 @@ def install( deffile=deffile, creation_date=datetime.now(), name=name, - tool=parsed_name.tool, - registry=parsed_name.registry, - repository=parsed_name.repository, - envfile=self.settings.environment_file, - wrapper_scripts=self.settings.wrapper_scripts, - wrapper_subdir=self.wrapper_subdir, + parsed_name=parsed_name, + wrapper_scripts=wrapper_scripts, ) shpc.utils.write_file(module_path, out) @@ -354,7 +352,7 @@ def test_script(self, image, test_script): """ Given a test file, run it and respond accordingly. """ - command = ["singularity", "exec", image, "/bin/bash", test_script] + command = [self.command, "exec", image, "/bin/bash", test_script] result = shpc.utils.run_command(command) # We can't run on incompatible hosts diff --git a/shpc/main/modules/__init__.py b/shpc/main/modules/__init__.py index 4c71ab64b..281abb1bc 100644 --- a/shpc/main/modules/__init__.py +++ b/shpc/main/modules/__init__.py @@ -79,10 +79,6 @@ def modulefile(self): def templatefile(self): return "%s.%s" % (self.container.templatefile, self.module_extension) - @property - def wrappertemplatefile(self): - return "%s.%s" % (self.container.templatefile, "sh") - def uninstall(self, name, force=False): """ Given a unique resource identifier, uninstall a module @@ -151,8 +147,7 @@ def add(self, sif, module_name, **kwargs): module_name = self.add_namespace(module_name) template = self._load_template(self.templatefile) modulefile = os.path.join(self.settings.module_base, module_name.replace(":", os.sep), self.modulefile) - wrapper_template = self._load_template(self.wrappertemplatefile) - self.container.add(sif, module_name, modulefile, template, wrapper_template, **kwargs) + self.container.add(sif, module_name, modulefile, template, **kwargs) def get(self, module_name, env_file=False): """ @@ -330,7 +325,6 @@ def install(self, name, tag=None, **kwargs): # Get the template based on the module and container type template = self._load_template(self.templatefile) module_path = os.path.join(module_dir, self.modulefile) - wrapper_template = self._load_template(self.wrappertemplatefile) # If the module has a version, overrides version version = tag.name @@ -344,13 +338,13 @@ def install(self, name, tag=None, **kwargs): name, template, aliases=config.get_aliases(), + config=config, parsed_name=config.name, url=config.url, description=config.description, version=version, config_features=config.features, features=kwargs.get("features"), - wrapper_template=wrapper_template, ) # If the container tech does not need storage, clean up diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua index 2c559530e..145a638f1 100644 --- a/shpc/main/modules/templates/docker.lua +++ b/shpc/main/modules/templates/docker.lua @@ -16,16 +16,16 @@ Container: Commands include: - {|module_name|}-run: - {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}-v ${PWD} -w ${PWD} "$@" + {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}-v ${PWD} -w ${PWD} "$@" - {|module_name|}-shell: - {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}--entrypoint {{ shell }} -v ${PWD} -w ${PWD} + {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}--entrypoint {{ shell }} -v ${PWD} -w ${PWD} - {|module_name|}-exec: - {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint "" {% if envfile %}--env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} "$@" + {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint "" {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} "$@" - {|module_name|}-inspect: {{ command }} inspect {% if aliases %}{% for alias in aliases %} - {{ alias.name }}: - {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ alias.entrypoint }} {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options }} {% endif %} -v ${PWD} -w ${PWD} "{{ alias.args }}" "$@" + {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ alias.entrypoint }} {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options }} {% endif %} -v ${PWD} -w ${PWD} "{{ alias.args }}" "$@" {% endfor %}{% endif %} For each of the above, you can export: @@ -34,7 +34,7 @@ For each of the above, you can export: - PODMAN_COMMAND_OPTS: to define custom options for the command ]]) -{% if podman_module %}load("{{ podman_module }}"){% endif %} +{% if settings.podman_module %}load("{{ settings.podman_module }}"){% endif %} -- Environment: only set options and command options if not already set if not os.getenv("PODMAN_OPTS") then setenv ("PODMAN_OPTS", "") end @@ -46,28 +46,30 @@ local MODULEPATH="{{ module_dir }}" -- interactive shell to any container, plus exec for aliases local containerPath = '{{ image }}' -local shellCmd = "{{ command }} ${PODMAN_OPTS} run -i{% if tty %}t{% endif %} ${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath +local shellCmd = "{{ command }} ${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} ${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath -- execCmd needs entrypoint to be the executor -local execCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " -local runCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath +local execCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " +local runCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath local inspectCmd = "{{ command }} ${PODMAN_OPTS} inspect ${PODMAN_COMMAND_OPTS} " .. containerPath -- set_shell_function takes bashStr and cshStr set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) -- conflict with modules with the same name -conflict("{{ tool }}"{% if name != tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) +conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) --- "aliases" to module commands -{% if wrapper_scripts %}{% if aliases %}prepend_path("PATH", pathJoin(myFileName():match("(.*[/])") or ".", "{{ wrapper_subdir }}")){% endif %} -{% else %}{% if aliases %}{% for alias in aliases %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }} \"$@\"", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }}") +-- if we have any wrapper scripts, add the bin directory +{% if wrapper_scripts %}prepend_path("PATH", pathJoin(myFileName():match("(.*[/])") or ".", "{{ settings.wrapper_subdir }}")){% endif %} + +-- "aliases" to module commands - generate only if not a wrapper script already generated +{% if aliases %}{% for alias in aliases %}{% if alias.name not in wrapper_scripts %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }} \"$@\"", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }}"){% endif %} {% endfor %}{% endif %} {% if aliases %} if (myShellName() == "bash") then -{% for alias in aliases %}execute{cmd="export -f {{ alias.name }}", modeA={"load"}} +{% for alias in aliases %}{% if alias.name not in wrapper_scripts %}execute{cmd="export -f {{ alias.name }}", modeA={"load"}}{% endif %} {% endfor %} -end{% endif %}{% endif %} +end{% endif %} -- A customizable exec function set_shell_function("{|module_name|}-exec", execCmd .. " --entrypoint \"\" " .. containerPath .. " \"$@\"", execCmd .. " --entrypoint \"\" " .. containerPath) diff --git a/shpc/main/modules/templates/docker.sh b/shpc/main/modules/templates/docker.sh deleted file mode 100755 index 4e4f61bee..000000000 --- a/shpc/main/modules/templates/docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!{{ wrapper_shell }} - -{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} --entrypoint {{ alias.entrypoint }} {{ image }} {{ alias.args }} {% if '/sh' in wrapper_shell or '/bash' in wrapper_shell %}"$@"{% elif '/csh' in wrapper_shell %}$argv:q{% endif %} diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl index 9d71f8c0e..40e58419d 100644 --- a/shpc/main/modules/templates/docker.tcl +++ b/shpc/main/modules/templates/docker.tcl @@ -15,16 +15,16 @@ proc ModulesHelp { } { puts stderr " - {{ image }}" puts stderr "Commands include:" puts stderr " - {|module_name|}-run:" - puts stderr " {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . \"\$@\"" + puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . \"\$@\"" puts stderr " - {|module_name|}-shell:" - puts stderr " {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if envfile %} --env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . " + puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %} --env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . " puts stderr " - {|module_name|}-exec:" - puts stderr " {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\" {% if envfile %} --env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . \"\$@\"" + puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\" {% if settings.environment_file %} --env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . \"\$@\"" puts stderr " - {|module_name|}-inspect:" puts stderr " {{ command }} inspect " puts stderr "" {% if aliases %}{% for alias in aliases %} puts stderr " - {{ alias.name }}:" - puts stderr " {{ command }} run -i{% if tty %}t{% endif %} --rm -u `id -u`:`id -g` --entrypoint {{ alias.entrypoint | replace("$", "\$") }} {% if envfile %}--envfile {{ module_dir }}/{{ envfile }} {% endif %}{% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options | replace("$", "\$") }} {% endif %} -v . -w . {{ alias.args | replace("$", "\$") }} \"\$@\"" + puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} --rm -u `id -u`:`id -g` --entrypoint {{ alias.entrypoint | replace("$", "\$") }} {% if settings.environment_file %}--settings.environment_file {{ module_dir }}/{{ settings.environment_file }} {% endif %}{% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options | replace("$", "\$") }} {% endif %} -v . -w . {{ alias.args | replace("$", "\$") }} \"\$@\"" {% endfor %}{% endif %} puts stderr "" puts stderr "For each of the above, you can export:" @@ -57,37 +57,39 @@ set helpcommand "This module is a {{ docker }} container wrapper for {{ name }} {% endfor %}{% endif %} # conflict with modules with the same alias name -conflict {{ tool }} -{% if name != tool %}conflict {{ name }}{% endif %} -{% if aliases %}{% for alias in aliases %}{% if alias.name != tool %}conflict {{ alias.name }}{% endif %} +conflict {{ parsed_name.tool }} +{% if name != parsed_name.tool %}conflict {{ name }}{% endif %} +{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %}conflict {{ alias.name }}{% endif %} {% endfor %}{% endif %} # interactive shell to any container, plus exec for aliases -set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if tty %}t{% endif %} --entrypoint {{ shell }} {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" +set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if settings.enable_tty %}t{% endif %} --entrypoint {{ shell }} {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" # execCmd needs entrypoint to be the executor -set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if envfile %} --env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }}{% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir" -set runCmd "{{ command }} \${PODMAN_OPTS} run -i{% if tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" +set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if settings.environment_file %} --env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }}{% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir" +set runCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}" # set_shell_function takes bashStr and cshStr set-alias {|module_name|}-shell "${shellCmd}" +# wrapper scripts? Add bin to path +{% if wrapper_scripts %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/{{ settings.wrapper_subdir }}"{% endif %} + # "aliases" to module commands -{% if wrapper_scripts %}{% if aliases %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/{{ wrapper_subdir }}"{% endif %} -{% else %}{% if aliases %}if { [ module-info shell bash ] } { +{% if aliases %}if { [ module-info shell bash ] } { if { [ module-info mode load ] } { -{% for alias in aliases %} puts stdout "function {{ alias.name }}() { ${execCmd} {% if alias.docker_options %} {{ alias.docker_options | replace("$", "\$") }} {% endif %} --entrypoint {{ alias.entrypoint | replace("$", "\$") }} ${containerPath} {{ alias.args | replace("$", "\$") }} \"\$@\"; }; export -f {{ alias.name }};" +{% for alias in aliases %}{% if alias.name not in wrapper_scripts %} puts stdout "function {{ alias.name }}() { ${execCmd} {% if alias.docker_options %} {{ alias.docker_options | replace("$", "\$") }} {% endif %} --entrypoint {{ alias.entrypoint | replace("$", "\$") }} ${containerPath} {{ alias.args | replace("$", "\$") }} \"\$@\"; }; export -f {{ alias.name }};"{% endif %} {% endfor %} } if { [ module-info mode remove ] } { -{% for alias in aliases %} puts stdout "unset -f {{ alias.name }};" +{% for alias in aliases %}{% if alias.name not in wrapper_scripts %} puts stdout "unset -f {{ alias.name }};"{% endif %} {% endfor %} } } else { -{% for alias in aliases %} set-alias {{ alias.name }} "${execCmd} {% if alias.docker_options %} {{ alias.docker_options | replace("$", "\$") }} {% endif %} --entrypoint {{ alias.entrypoint | replace("$", "\$") }} ${containerPath} {{ alias.args | replace("$", "\$") }}" +{% for alias in aliases %}{% if alias.name not in wrapper_scripts %} set-alias {{ alias.name }} "${execCmd} {% if alias.docker_options %} {{ alias.docker_options | replace("$", "\$") }} {% endif %} --entrypoint {{ alias.entrypoint | replace("$", "\$") }} ${containerPath} {{ alias.args | replace("$", "\$") }}"{% endif %} {% endfor %} -}{% endif %}{% endif %} +}{% endif %} # A customizable exec function if { [ module-info shell bash ] } { @@ -115,4 +117,4 @@ module-whatis " Version: {{ version }}" {% if url %}module-whatis " Url: {{ url }}"{% endif %} {% if labels %}{% for key, value in labels.items() %}module-whatis " {{ key }}: {{ value }}" {% endfor %}{% endif %} -{% if podman_module %}module load {{ podman_module }}{% endif %} +{% if settings.podman_module %}module load {{ settings.podman_module }}{% endif %} diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index d3674ce33..254832034 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -16,18 +16,18 @@ Container: Commands include: - {|module_name|}-run: - singularity run {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }} {% endif %} "$@" + singularity run {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} "$@" - {|module_name|}-shell: - singularity shell -s {{ singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }} {% endif %} + singularity shell -s {{ singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} - {|module_name|}-exec: - singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }} {% endif %} "$@" + singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} "$@" - {|module_name|}-inspect-runscript: singularity inspect -r - {|module_name|}-inspect-deffile: singularity inspect -d {% if aliases %}{% for alias in aliases %} - {{ alias.name }}: - singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options }} {% endif %} {{ alias.command }} "$@" + singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options }} {% endif %} {{ alias.command }} "$@" {% endfor %}{% endif %} For each of the above, you can export: @@ -36,13 +36,13 @@ For each of the above, you can export: - SINGULARITY_COMMAND_OPTS: to define custom options for the command (e.g., -b) ]]) -{% if singularity_module %}load("{{ singularity_module }}"){% endif %} +{% if settings.singularity_module %}load("{{ settings.singularity_module }}"){% endif %} -- we probably do not need this local MODULEPATH="{{ module_dir }}" -- singularity environment variable to set shell -setenv("SINGULARITY_SHELL", "{{ singularity_shell }}") +setenv("SINGULARITY_SHELL", "{{ settings.singularity_shell }}") -- Environment: only set options and command options if not already set if not os.getenv("SINGULARITY_OPTS") then setenv ("SINGULARITY_OPTS", "") end @@ -50,27 +50,29 @@ if not os.getenv("SINGULARITY_COMMAND_OPTS") then setenv ("SINGULARITY_COMMAND_O -- interactive shell to any container, plus exec for aliases local containerPath = '{{ container_sif }}' -local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} " .. containerPath -local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} " -local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} " .. containerPath +local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath +local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " +local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath local inspectCmd = "singularity ${SINGULARITY_OPTS} inspect ${SINGULARITY_COMMAND_OPTS} " -- set_shell_function takes bashStr and cshStr set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) -- conflict with modules with the same name -conflict("{{ tool }}"{% if name != tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) +conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) + +-- if we have any wrapper scripts, add bin to path +{% if wrapper_scripts %}prepend_path("PATH",pathJoin(myFileName():match("(.*[/])") or ".", "{{ settings.wrapper_subdir }}")){% endif %} -- "aliases" to module commands -{% if wrapper_scripts %}{% if aliases %}prepend_path("PATH",pathJoin(myFileName():match("(.*[/])") or ".", "{{ wrapper_subdir }}")){% endif %} -{% else %}{% if aliases %}{% for alias in aliases %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }} \"$@\"", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }}") +{% if aliases %}{% for alias in aliases %}{% if alias.name not in wrapper_scripts %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }} \"$@\"", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }}"){% endif %} {% endfor %}{% endif %} {% if aliases %} if (myShellName() == "bash") then -{% for alias in aliases %}execute{cmd="export -f {{ alias.name }}", modeA={"load"}} +{% for alias in aliases %}{% if alias.name not in wrapper_scripts %}execute{cmd="export -f {{ alias.name }}", modeA={"load"}}{% endif %} {% endfor %} -end{% endif %}{% endif %} +end{% endif %} -- A customizable exec function set_shell_function("{|module_name|}-exec", execCmd .. containerPath .. " \"$@\"", execCmd .. containerPath) diff --git a/shpc/main/modules/templates/singularity.sh b/shpc/main/modules/templates/singularity.sh deleted file mode 100755 index 9b12aa58d..000000000 --- a/shpc/main/modules/templates/singularity.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!{{ wrapper_shell }} - -singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} {% if alias.singularity_options %} {{ alias.singularity_options }} {% endif %} {{ container_sif }} {{ alias.command }} {% if '/sh' in wrapper_shell or '/bash' in wrapper_shell %}"$@"{% elif '/csh' in wrapper_shell %}$argv:q{% endif %} \ No newline at end of file diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl index a69868212..4484821fd 100644 --- a/shpc/main/modules/templates/singularity.tcl +++ b/shpc/main/modules/templates/singularity.tcl @@ -18,18 +18,18 @@ proc ModulesHelp { } { puts stderr "Commands include:" puts stderr "" puts stderr " - {|module_name|}-run:" - puts stderr " singularity run {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }} {% endif %} \"\$@\"" + puts stderr " singularity run {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} \"\$@\"" puts stderr " - {|module_name|}-shell:" - puts stderr " singularity shell -s {{ singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }} {% endif %}" + puts stderr " singularity shell -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}" puts stderr " - {|module_name|}-exec:" - puts stderr " singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }} {% endif %} \"\$@\"" + puts stderr " singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} \"\$@\"" puts stderr " - {|module_name|}-inspect-runscript:" puts stderr " singularity inspect -r " puts stderr " - {|module_name|}-inspect-deffile:" puts stderr " singularity inspect -d " puts stderr "" {% if aliases %}{% for alias in aliases %} puts stderr " - {{ alias.name }}:" - puts stderr " singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options | replace("$", "\$") }} {% endif %} {{ alias.command | replace("$", "\$") }} \"\$@\"" + puts stderr " singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options | replace("$", "\$") }} {% endif %} {{ alias.command | replace("$", "\$") }} \"\$@\"" {% endfor %}{% endif %} puts stderr "" puts stderr "For each of the above, you can export:" @@ -61,38 +61,41 @@ set helpcommand "This module is a singularity container wrapper for {{ name }} v {% endfor %}{% endif %} # conflict with modules with the same alias name -conflict {{ tool }} -{% if name != tool %}conflict {{ name }}{% endif %} -{% if aliases %}{% for alias in aliases %}{% if alias.name != tool %}conflict {{ alias.name }}{% endif %} +conflict {{ parsed_name.tool }} +{% if name != parsed_name.tool %}conflict {{ name }}{% endif %} +{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %}conflict {{ alias.name }}{% endif %} {% endfor %}{% endif %} # singularity environment variable to set shell -setenv SINGULARITY_SHELL {{ singularity_shell }} +setenv SINGULARITY_SHELL {{ settings.singularity_shell }} # interactive shell to any container, plus exec for aliases -set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s {{ singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} ${containerPath}" -set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} " -set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if envfile %}-B {{ module_dir }}/{{ envfile }}:/.singularity.d/env/{{ envfile }}{% endif %} {% if bindpaths %}-B {{ bindpaths }}{% endif %} ${containerPath}" +set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}" +set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " +set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}" set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_OPTS} " # set_shell_function takes bashStr and cshStr set-alias {|module_name|}-shell "${shellCmd}" + +# if we have any wrapper scripts, add bin to path +{% if wrapper_scripts %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/{{ settings.wrapper_subdir }}"{% endif %} + # "aliases" to module commands -{% if wrapper_scripts %}{% if aliases %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/{{ wrapper_subdir }}"{% endif %} -{% else %}{% if aliases %}if { [ module-info shell bash ] } { +{% if aliases %}if { [ module-info shell bash ] } { if { [ module-info mode load ] } { -{% for alias in aliases %} puts stdout "function {{ alias.name }}() { ${execCmd} {% if alias.singularity_options %} {{ alias.singularity_options | replace("$", "\$") }} {% endif %} ${containerPath} {{ alias.command | replace("$", "\$") }} \"\$@\"; }; export -f {{ alias.name }};" +{% for alias in aliases %} {% if alias.name not in wrapper_scripts %} puts stdout "function {{ alias.name }}() { ${execCmd} {% if alias.singularity_options %} {{ alias.singularity_options | replace("$", "\$") }} {% endif %} ${containerPath} {{ alias.command | replace("$", "\$") }} \"\$@\"; }; export -f {{ alias.name }};"{% endif %} {% endfor %} } if { [ module-info mode remove ] } { -{% for alias in aliases %} puts stdout "unset -f {{ alias.name }};" +{% for alias in aliases %} {% if alias.name not in wrapper_scripts %} puts stdout "unset -f {{ alias.name }};"{% endif %} {% endfor %} } } else { -{% for alias in aliases %} set-alias {{ alias.name }} "${execCmd} {% if alias.singularity_options %} {{ alias.singularity_options | replace("$", "\$") }} {% endif %} ${containerPath} {{ alias.command | replace("$", "\$") }}" +{% for alias in aliases %}{% if alias.name not in wrapper_scripts %} set-alias {{ alias.name }} "${execCmd} {% if alias.singularity_options %} {{ alias.singularity_options | replace("$", "\$") }} {% endif %} ${containerPath} {{ alias.command | replace("$", "\$") }}"{% endif %} {% endfor %} -}{% endif %}{% endif %} +}{% endif %} # A customizable exec function if { [ module-info shell bash ] } { @@ -112,6 +115,7 @@ if { [ module-info shell bash ] } { set-alias {|module_name|}-inspect-runscript "${inspectCmd} -r ${containerPath}" set-alias {|module_name|}-inspect-deffile "${inspectCmd} -d ${containerPath}" + #===== # Module options #===== @@ -121,4 +125,4 @@ module-whatis " Version: {{ version }}" {% if url %}module-whatis " Url: {{ url }}"{% endif %} {% if labels %}{% for key, value in labels.items() %}module-whatis " {{ key }}: {{ value }}" {% endfor %}{% endif %} -{% if singularity_module %}module load {{ singularity_module }}{% endif %} +{% if settings.singularity_module %}module load {{ settings.singularity_module }}{% endif %} diff --git a/shpc/main/schemas.py b/shpc/main/schemas.py index 5deaa0efc..d458d2829 100644 --- a/shpc/main/schemas.py +++ b/shpc/main/schemas.py @@ -10,7 +10,7 @@ # This is also for latest, and a list of tags # The simplest form of aliases is key/value pairs -aliases = { +keyvals = { "type": "object", "patternProperties": { "\\w[\\w-]*": {"type": "string"}, @@ -67,7 +67,7 @@ } containerConfigProperties = { - "latest": aliases, + "latest": keyvals, "docker": {"type": "string"}, "oras": {"type": "string"}, "gh": {"type": "string"}, @@ -75,16 +75,18 @@ "test": {"type": "string"}, "maintainer": {"type": "string"}, "description": {"type": "string"}, - "tags": aliases, + "docker_scripts": keyvals, + "singularity_scripts": keyvals, + "tags": keyvals, "filter": { "type": "array", "items": {"type": "string"}, }, - "env": aliases, + "env": keyvals, "features": features, "aliases": { "oneOf": [ - aliases, + keyvals, aliases_list, ] }, @@ -105,6 +107,17 @@ "additionalProperties": False, } +# Wrapper scripts for global (aliases) and container.yaml +wrapper_scripts = { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "docker": {"type": "string"}, + "podman": {"type": "string"}, + "singularity": {"type": "string"}, + }, +} + ## Settings.yml (loads as json) @@ -125,7 +138,8 @@ "environment_file": {"type": "string"}, "default_version": {"type": "boolean"}, "enable_tty": {"type": "boolean"}, - "wrapper_scripts": {"type": "boolean"}, + "wrapper_scripts": wrapper_scripts, + "wrapper_subdir": {"type": "string"}, "container_tech": {"type": "string", "enum": ["singularity", "podman", "docker"]}, "singularity_shell": {"type": "string", "enum": shells}, "podman_shell": {"type": "string", "enum": shells}, diff --git a/shpc/main/wrappers/__init__.py b/shpc/main/wrappers/__init__.py new file mode 100644 index 000000000..d60ecb4ef --- /dev/null +++ b/shpc/main/wrappers/__init__.py @@ -0,0 +1,62 @@ +__author__ = "Vanessa Sochat" +__copyright__ = "Copyright 2022, Vanessa Sochat" +__license__ = "MPL 2.0" + +from .scripts import get_wrapper_script +import os + + +here = os.path.abspath(os.path.dirname(__file__)) + + +def generate(image, container, config, **kwargs): + """ + Generate one or more wrapper scripts for a container. + + Required arguments for all include container (class), image (path), settings + All kwargs go in optional. And this can be extended to include custom arguments. + """ + # Return list of generated templates + generated = [] + + settings = container.settings + aliases = kwargs.get("aliases", []) + + # Global wrapper scripts are generated via aliases + for container_tech in ["docker", "podman", "singularity"]: + template_name = settings.wrapper_scripts.get(container_tech) + if template_name and aliases and container_tech == container.command: + wrapper = get_wrapper_script(template_name)( + settings=settings, + wrapper_template=template_name, + config=config, + container=container, + image=image, + **kwargs + ) + generated += wrapper.generate_aliases() + + # Container level wrapper scripts (allow eventually supporting custom podman) + scripts = { + "singularity": config.singularity_scripts, + "docker": config.docker_scripts, + "podman": config.docker_scripts, + } + for command, listing in scripts.items(): + + # Don't look if no custom container.yaml scripts OR wrong container tech + if not listing or container.command != command: + continue + for alias, template_name in listing.items(): + wrapper = get_wrapper_script(template_name)( + settings=settings, + wrapper_template=template_name, + config=config, + container=container, + image=image, + **kwargs + ) + generated += wrapper.generate(alias) + + print(generated) + return list(set(generated)) diff --git a/shpc/main/wrappers/base.py b/shpc/main/wrappers/base.py new file mode 100644 index 000000000..7f2034a50 --- /dev/null +++ b/shpc/main/wrappers/base.py @@ -0,0 +1,136 @@ +__author__ = "Vanessa Sochat" +__copyright__ = "Copyright 2022, Vanessa Sochat" +__license__ = "MPL 2.0" + +from shpc.logger import logger +from jinja2 import Template +import shpc.utils +import os + +here = os.path.dirname(os.path.abspath(__file__)) + + +class WrapperScript: + """ + The base class of a wrapper script provides basic wrapper script functionality, + which can be extended for custom named wrapper scripts. + """ + + def __init__( + self, + settings, + image, + container=None, + config=None, + wrapper_template=None, + **kwargs + ): + """ + On init, a wrapper_template should be provided (string) OR the class + should have an attribute. + """ + self.settings = settings + self.container = container + self.config = config + self.kwargs = kwargs + self.image = image + self.template_type = "custom" + if not wrapper_template and not hasattr(self, "wrapper_template"): + logger.exit("A wrapper template is required to generate a wrapper script.") + if not hasattr(self, "wrapper_template") and wrapper_template: + self.wrapper_template = wrapper_template + + @property + def container_dest_dir(self): + return os.path.dirname(self.image) + + @property + def container_dir(self): + """ + Source of container.yaml + """ + return os.path.dirname(self.config.package_file) + + def load_template(self): + """ + Load the wrapper template. + """ + # Contender with the container dir + contender = os.path.join(self.container_dir, self.wrapper_template) + + # First shot - it already exists + template_file = None + if os.path.exists(self.wrapper_template): + template_file = self.wrapper_template + + # Second shot - it belongs with the container + elif os.path.exists(contender): + template_file = contender + + # Finally, it has to be in the template directory + if not template_file: + template_file = os.path.join(here, self.wrapper_template) + + if not os.path.exists(template_file): + logger.exit( + "%s not found as default, container, or custom script." + % self.wrapper_template + ) + + with open(template_file, "r") as temp: + template = Template(temp.read()) + self.template_file = template_file + self.template = template + + def generate_aliases(self): + """ + Generate looping over aliases + """ + self.load_template() + + # Write scripts into container directory + wrapper_dir = os.path.join( + self.container_dest_dir, self.settings.wrapper_subdir + ) + shpc.utils.mkdirp([wrapper_dir]) + + generated = [] + # When we get here we know we have aliases! + for alias in self.kwargs["aliases"]: + wrapper_path = os.path.join(wrapper_dir, alias["name"]) + self._generate(wrapper_path, alias) + generated.append(os.path.basename(wrapper_path)) + + return generated + + def _generate(self, wrapper_path, alias): + """ + Shared generation function. + """ + out = self.template.render( + alias=alias, + container=self.container, + settings=self.settings, + image=self.image, + config=self.config, + # includes module_dir, features, aliases + **self.kwargs + ) + shpc.utils.write_file(wrapper_path, out, exec=True) + + # Return the alias / script name + return os.path.basename(wrapper_path) + + def generate(self, alias): + """ + A default generation cannot know what the template wants, so we give it everything. + """ + self.load_template() + + # Write scripts into container directory + wrapper_dir = os.path.join( + self.container_dest_dir, self.settings.wrapper_subdir + ) + shpc.utils.mkdirp([wrapper_dir]) + wrapper_path = os.path.join(wrapper_dir, alias) + return [self._generate(wrapper_path, alias)] diff --git a/shpc/main/wrappers/docker.sh b/shpc/main/wrappers/docker.sh new file mode 100755 index 000000000..7afbdb1ba --- /dev/null +++ b/shpc/main/wrappers/docker.sh @@ -0,0 +1,3 @@ +#!{{ settings.wrapper_shell }} + +{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} --entrypoint {{ alias.entrypoint }} {{ image }} {{ alias.args }} {% if '/sh' in wrapper_shell or '/bash' in wrapper_shell %}"$@"{% elif '/csh' in wrapper_shell %}$argv:q{% endif %} diff --git a/shpc/main/wrappers/scripts.py b/shpc/main/wrappers/scripts.py new file mode 100644 index 000000000..3642d2dbd --- /dev/null +++ b/shpc/main/wrappers/scripts.py @@ -0,0 +1,24 @@ +__author__ = "Vanessa Sochat" +__copyright__ = "Copyright 2022, Vanessa Sochat" +__license__ = "MPL 2.0" + + +from .base import WrapperScript + + +def get_wrapper_script(name): + return ( + { + "docker.sh": DockerWrapper, + "singularity.sh": SingularityWrapper, + "podman.sh": DockerWrapper, + } + ).get(name) or WrapperScript + + +class DockerWrapper(WrapperScript): + wrapper_template = "docker.sh" + + +class SingularityWrapper(WrapperScript): + wrapper_template = "singularity.sh" diff --git a/shpc/main/wrappers/singularity.sh b/shpc/main/wrappers/singularity.sh new file mode 100755 index 000000000..baedb5deb --- /dev/null +++ b/shpc/main/wrappers/singularity.sh @@ -0,0 +1,3 @@ +#!{{ settings.wrapper_shell }} + +singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {% if alias.singularity_options %} {{ alias.singularity_options }} {% endif %} {{ image }} {{ alias.command }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} diff --git a/shpc/settings.yml b/shpc/settings.yml index e40d7b371..70b4ae9fa 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -51,11 +51,27 @@ docker_shell: /bin/sh # shell for test.sh file test_shell: /bin/bash -# shell for wrapper shellscripts +# shell for wrapper shell scripts wrapper_shell: /bin/bash -# for container aliases, use wrapper shellscripts instead of shell aliases, defaults to false -wrapper_scripts: false +# Wrapper scripts to use for aliases (replaces them) +# Put a fullpath here for a custom script, otherwise we look in shpc/main/wrappers +# If you want a custom wrapper script for a one-off container, define it there with the script +# stored alongside the container.yaml. +wrapper_scripts: + enabled: true + + # use for docker aliases (set to null to disable) + docker: docker.sh + + # use for podman aliases (set to null to disable) + podman: docker.sh + + # use for singularity aliases (set to null to disable) + singularity: singularity.sh + +# Wrapper scripts are stored in this subdirectory of the module directory +wrapper_subdir: "bin" # the module namespace you want to install from. E.g., if you have ghcr.io/autamus/clingo # and you set the namespace to ghcr.io/autamus, you can just do: shpc install clingo. diff --git a/shpc/version.py b/shpc/version.py index de02b19e2..ad6095014 100644 --- a/shpc/version.py +++ b/shpc/version.py @@ -2,7 +2,7 @@ __copyright__ = "Copyright 2021-2022, Vanessa Sochat" __license__ = "MPL 2.0" -__version__ = "0.0.44" +__version__ = "0.0.45" AUTHOR = "Vanessa Sochat" NAME = "singularity-hpc" PACKAGE_URL = "https://github.com/singularityhub/singularity-hpc" From 02f425f9462fb85b734c238ec243ad080c0c60b8 Mon Sep 17 00:00:00 2001 From: vsoch Date: Wed, 16 Feb 2022 02:55:49 -0700 Subject: [PATCH 25/38] enabled should be false Signed-off-by: vsoch --- shpc/settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shpc/settings.yml b/shpc/settings.yml index 70b4ae9fa..5c5e0505c 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -59,7 +59,7 @@ wrapper_shell: /bin/bash # If you want a custom wrapper script for a one-off container, define it there with the script # stored alongside the container.yaml. wrapper_scripts: - enabled: true + enabled: false # use for docker aliases (set to null to disable) docker: docker.sh From 99e7ac603a499b2cbb9cff12a1f7863385cbe41c Mon Sep 17 00:00:00 2001 From: vsoch Date: Wed, 16 Feb 2022 03:02:02 -0700 Subject: [PATCH 26/38] fix nonetypes not iterable issue Signed-off-by: vsoch --- shpc/main/container/docker.py | 2 +- shpc/main/container/singularity.py | 2 +- shpc/main/wrappers/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shpc/main/container/docker.py b/shpc/main/container/docker.py index d32a177c1..716a6fc90 100644 --- a/shpc/main/container/docker.py +++ b/shpc/main/container/docker.py @@ -218,7 +218,7 @@ def install( # Option to create wrapper scripts for commands module_dir = os.path.dirname(module_path) - wrapper_scripts = None + wrapper_scripts = [] # Wrapper scripts can be global (for aliases) or container specific if self.settings.wrapper_scripts["enabled"] is True: diff --git a/shpc/main/container/singularity.py b/shpc/main/container/singularity.py index b7d8cc02a..b177fd5fc 100644 --- a/shpc/main/container/singularity.py +++ b/shpc/main/container/singularity.py @@ -172,7 +172,7 @@ def install( module_dir = os.path.dirname(module_path) # Wrapper scripts can be global (for aliases) or container specific - wrapper_scripts = None + wrapper_scripts = [] if self.settings.wrapper_scripts["enabled"] is True: wrapper_scripts = shpc.main.wrappers.generate( aliases=aliases, diff --git a/shpc/main/wrappers/__init__.py b/shpc/main/wrappers/__init__.py index d60ecb4ef..ed8b3404f 100644 --- a/shpc/main/wrappers/__init__.py +++ b/shpc/main/wrappers/__init__.py @@ -45,7 +45,7 @@ def generate(image, container, config, **kwargs): for command, listing in scripts.items(): # Don't look if no custom container.yaml scripts OR wrong container tech - if not listing or container.command != command: + if not listing or container.templatefile != command: continue for alias, template_name in listing.items(): wrapper = get_wrapper_script(template_name)( From 8affab3d14e90f49d5c34ffe0cb1e5c1394893ab Mon Sep 17 00:00:00 2001 From: vsoch Date: Wed, 16 Feb 2022 03:22:20 -0700 Subject: [PATCH 27/38] fixing invalid load command warning - parsed_names (tool, registry, etc) are now prefixed with parsed_name. and the user will need to update settings Signed-off-by: vsoch --- shpc/main/modules/__init__.py | 4 ++-- shpc/settings.yml | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/shpc/main/modules/__init__.py b/shpc/main/modules/__init__.py index 281abb1bc..a23168829 100644 --- a/shpc/main/modules/__init__.py +++ b/shpc/main/modules/__init__.py @@ -41,7 +41,7 @@ def substitute(self, template): """ For all known identifiers, substitute user specified format strings. """ - subs = {"{|module_name|}": self.settings.module_name or "{{ tool }}"} + subs = {"{|module_name|}": self.settings.module_name or "{{ parsed_name.tool }}"} for key, replacewith in subs.items(): template = template.replace(key, replacewith) return template @@ -188,7 +188,7 @@ def docgen(self, module_name, out=None): singularity_module=self.settings.singularity_module, # Show same shell for all container technologies shell=self.settings.singularity_shell, - bindpaths=self.settings.bindpaths, + bindpaths=self.settings.bindpaths, description=config.description, aliases=aliases, versions=config.tags.keys(), diff --git a/shpc/settings.yml b/shpc/settings.yml index 5c5e0505c..bdf2f6d52 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -22,9 +22,10 @@ registry: [$root_dir/registry] module_base: $root_dir/modules # Format string for module commands exec,run,shell (not aliases) (can also include: -# {{registry}} , {{ repository }}, {{tool}}, {{version}} +# {{parsed_name.registry}} , {{ parsed_name.repository }}, +# {{parsed_name.tool}}, {{parsed_name.version}} # This is where you might add a prefix to your module names, if desired. -module_name: '{{ tool }}' +module_name: '{{ parsed_name.tool }}' # Create a .version file for LMOD in the module folder default_version: true From 55c8e661030982e2b28d3e5b72165c8e120eef3f Mon Sep 17 00:00:00 2001 From: vsoch Date: Wed, 16 Feb 2022 03:27:11 -0700 Subject: [PATCH 28/38] docker/podman shell typo Signed-off-by: vsoch --- shpc/main/container/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shpc/main/container/docker.py b/shpc/main/container/docker.py index 716a6fc90..7a096def7 100644 --- a/shpc/main/container/docker.py +++ b/shpc/main/container/docker.py @@ -235,7 +235,7 @@ def install( shell = ( self.settings.podman_shell if self.command == "podman" - else self.settings.docker_shell, + else self.settings.docker_shell ) # Make sure to render all values! From 312b78f33fe633f312a9a853b9eb40197b66ee0e Mon Sep 17 00:00:00 2001 From: vsoch Date: Wed, 16 Feb 2022 03:42:18 -0700 Subject: [PATCH 29/38] adding parsed name Signed-off-by: vsoch --- shpc/main/modules/__init__.py | 6 ++--- shpc/main/modules/templates/docs.md | 35 +++++++++++++++-------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/shpc/main/modules/__init__.py b/shpc/main/modules/__init__.py index a23168829..b580972c6 100644 --- a/shpc/main/modules/__init__.py +++ b/shpc/main/modules/__init__.py @@ -185,10 +185,8 @@ def docgen(self, module_name, out=None): # Currently one doc is rendered for all containers result = template.render( - singularity_module=self.settings.singularity_module, - # Show same shell for all container technologies - shell=self.settings.singularity_shell, - bindpaths=self.settings.bindpaths, + parsed_name=config.name, + settings=self.settings, description=config.description, aliases=aliases, versions=config.tags.keys(), diff --git a/shpc/main/modules/templates/docs.md b/shpc/main/modules/templates/docs.md index 834f45ea1..2eb7579f0 100644 --- a/shpc/main/modules/templates/docs.md +++ b/shpc/main/modules/templates/docs.md @@ -17,6 +17,7 @@ This module is a singularity container wrapper for {{ name }}. {% if description %}{{ description }}{% endif %} After [installing shpc](#install) you will want to install this container module: + ```bash $ shpc install {{ name }} ``` @@ -52,25 +53,25 @@ Examples for both Singularity, Podman, and Docker (container technologies suppor #### {|module_name|}-run: ```bash -$ singularity run {% if bindpaths %}-B {{ bindpaths }} {% endif %} -$ podman run --rm {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v ${PWD} -w ${PWD} -$ docker run --rm {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v ${PWD} -w ${PWD} +$ singularity run {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} +$ podman run --rm {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} +$ docker run --rm {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} ``` -#### {|module_name|}-shell: +#### {|module_name|}-settings.singularity_shell: ```bash -$ singularity shell -s {{ shell }} {% if bindpaths %}-B {{ bindpaths }} {% endif %} -$ podman run --it --rm --entrypoint {{ shell }} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v ${PWD} -w ${PWD} -$ docker run --it --rm --entrypoint {{ shell }} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v ${PWD} -w ${PWD} +$ singularity settings.singularity_shell -s {{ settings.singularity_shell }} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} +$ podman run --it --rm --entrypoint {{ settings.singularity_shell }} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} +$ docker run --it --rm --entrypoint {{ settings.singularity_shell }} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} ``` #### {|module_name|}-exec: ```bash -$ singularity exec -s {{ shell }} {% if bindpaths %}-B {{ bindpaths }} {% endif %} "$@" -$ podman run --it --rm --entrypoint "" {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v ${PWD} -w ${PWD} "$@" -$ docker run --it --rm --entrypoint "" {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v ${PWD} -w ${PWD} "$@" +$ singularity exec -s {{ settings.singularity_shell }} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} "$@" +$ podman run --it --rm --entrypoint "" {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} "$@" +$ docker run --it --rm --entrypoint "" {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} "$@" ``` #### {|module_name|}-inspect: @@ -98,9 +99,9 @@ $ singularity inspect -d #### {{ alias.name }} ```bash -$ singularity exec {% if bindpaths %}-B {{ bindpaths }} {% endif %}{% if alias.options %}{{ alias.options }} {% endif %} {{ alias.command }} -$ podman run --it --rm --entrypoint {{ alias.entrypoint }} {% if bindpaths %}-v {{ bindpaths }} {% endif %} {% if alias.options %}{{ alias.options }} {% endif %} -v ${PWD} -w ${PWD} -c "{{ alias.args }} $@" -$ docker run --it --rm --entrypoint {{ alias.entrypoint }} {% if bindpaths %}-v {{ bindpaths }} {% endif %} {% if alias.options %}{{ alias.options }} {% endif %} -v ${PWD} -w ${PWD} -c "{{ alias.args }} $@" +$ singularity exec {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}{% if alias.options %}{{ alias.options }} {% endif %} {{ alias.command }} +$ podman run --it --rm --entrypoint {{ alias.entrypoint }} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} {% if alias.options %}{{ alias.options }} {% endif %} -v ${PWD} -w ${PWD} -c "{{ alias.args }} $@" +$ docker run --it --rm --entrypoint {{ alias.entrypoint }} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} {% if alias.options %}{{ alias.options }} {% endif %} -v ${PWD} -w ${PWD} -c "{{ alias.args }} $@" ``` {% endfor %}{% else %} @@ -108,16 +109,16 @@ $ docker run --it --rm --entrypoint {{ alias.entrypoint }} {% if bindpaths %}-v #### {|module_name|} ```bash -$ singularity run {% if bindpaths %}-B {{ bindpaths }}{% endif %} -$ podman run --rm {% if bindpaths %}-v {{ bindpaths }}{% endif %} -v ${PWD} -w ${PWD} -$ docker run --rm {% if bindpaths %}-v {{ bindpaths }}{% endif %} -v ${PWD} -w ${PWD} +$ singularity run {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} +$ podman run --rm {% if settings.bindpaths %}-v {{ settings.bindpaths }}{% endif %} -v ${PWD} -w ${PWD} +$ docker run --rm {% if settings.bindpaths %}-v {{ settings.bindpaths }}{% endif %} -v ${PWD} -w ${PWD} ``` {% endif %} In the above, the `` directive will reference an actual container provided by the module, for the version you have chosen to load. An environment file in the module folder will also be bound. Note that although a container -might provide custom commands, every container exposes unique exec, shell, run, and +might provide custom commands, every container exposes unique exec, settings.singularity_shell, run, and inspect aliases. For anycommands above, you can export: - SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug) From d88fbcd7f4ca8965812115b15e8308c66701fd01 Mon Sep 17 00:00:00 2001 From: vsoch Date: Sun, 20 Feb 2022 20:59:38 -0700 Subject: [PATCH 30/38] updating to install to module directory and adding example to docs Signed-off-by: vsoch --- docs/getting_started/user-guide.rst | 40 +++++++++++++++++++++++++---- shpc/main/wrappers/__init__.py | 1 - shpc/main/wrappers/base.py | 15 ++++++----- shpc/settings.yml | 2 +- 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst index 12e0021d8..d2e5b4c45 100644 --- a/docs/getting_started/user-guide.rst +++ b/docs/getting_started/user-guide.rst @@ -453,11 +453,6 @@ Singularity HPC allows for global definition of wrapper scripts, meaning that in we generate a wrapper script of the same name instead. Since the settings.yml is global, all wrapper scripts defined here are specific to replacing aliases. Container-specific scripts you'll want to include in the container.yaml, described in the developer docs. Let's take a look at the settings: -# Wrapper scripts to use for aliases (replaces them) -# Put a fullpath here for a custom script, otherwise we look in shpc/main/wrappers -# If you want a custom wrapper script for a one-off container, define it there with the script -# stored alongside the container.yaml. - .. code-block:: yaml @@ -476,6 +471,7 @@ Container-specific scripts you'll want to include in the container.yaml, describ # use for singularity aliases singularity: singularity.sh + Since different container technologies might expose different environment variables (e.g., ``SINGULARITY_OPTS`` vs ``PODMAN_OPTS``) they are organized above based on the container technology. If you want to customize the wrapper script, simply replace the relative paths above (e.g., ``singularity.sh`` with an absolute path to a file that will be used instead. For global alias scripts such as these, @@ -484,6 +480,19 @@ Singularity HPC will look for: 1. An absolute path first, if found is used first. 2. Then a script name in the shpc/main/wrappers directory +Here is an example of using wrapper scripts for the "python" container, which doesn't have container specific wrappers. What you see +is the one entrypoint, "python" being placed in a bin that the module will see instead of defining the alias. + + +.. code-block:: console + + modules/python/ + └── 3.9.10 + ├── 99-shpc.sh + ├── bin + │   └── python + └── module.lua + For container specific scripts, you can add sections to a ``container.yaml`` to specify the script (and container type) and the scripts must be provided alongside the container.yaml to install. @@ -504,6 +513,27 @@ and the same for Singularity. And then I (the developer) would provide the custo ├── docker_fork.sh └── singularity_fork.sh +And here is what those scripts look like installed. Since we are installing for just one container technology, we are seeing the alias wrapper for salad as "salad" and the container-specific wrapper for fork as "fork." + + +.. code-block:: console + + modules/vanessa/salad/ + └── latest + ├── 99-shpc.sh + ├── bin + │   ├── fork + │   └── salad + └── module.lua + + +We currently don't have a global argument to enable alias wrappers but not container wrappers. If you see a need for this please let us know. + +Where are wrapper scripts stored? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Since we don't allow overlap +of the name of an alias wrapper script (e.g., ``bin/python``) as a wrapper to a python entrypoint) from a custom container wrapper script (e.g., a wrapper script with name "python" under a container.yaml) we can keep them both in the modules directory. If you see a need to put them elsewhere please let us kno. .. _getting_started-commands: diff --git a/shpc/main/wrappers/__init__.py b/shpc/main/wrappers/__init__.py index ed8b3404f..b60e06727 100644 --- a/shpc/main/wrappers/__init__.py +++ b/shpc/main/wrappers/__init__.py @@ -58,5 +58,4 @@ def generate(image, container, config, **kwargs): ) generated += wrapper.generate(alias) - print(generated) return list(set(generated)) diff --git a/shpc/main/wrappers/base.py b/shpc/main/wrappers/base.py index 7f2034a50..847a47746 100644 --- a/shpc/main/wrappers/base.py +++ b/shpc/main/wrappers/base.py @@ -51,6 +51,13 @@ def container_dir(self): """ return os.path.dirname(self.config.package_file) + @property + def module_dir(self): + """ + Get the module directory (should error if not provided in kwargs) + """ + return self.kwargs["module_dir"] + def load_template(self): """ Load the wrapper template. @@ -89,9 +96,7 @@ def generate_aliases(self): self.load_template() # Write scripts into container directory - wrapper_dir = os.path.join( - self.container_dest_dir, self.settings.wrapper_subdir - ) + wrapper_dir = os.path.join(self.module_dir, self.settings.wrapper_subdir) shpc.utils.mkdirp([wrapper_dir]) generated = [] @@ -128,9 +133,7 @@ def generate(self, alias): self.load_template() # Write scripts into container directory - wrapper_dir = os.path.join( - self.container_dest_dir, self.settings.wrapper_subdir - ) + wrapper_dir = os.path.join(self.module_dir, self.settings.wrapper_subdir) shpc.utils.mkdirp([wrapper_dir]) wrapper_path = os.path.join(wrapper_dir, alias) return [self._generate(wrapper_path, alias)] diff --git a/shpc/settings.yml b/shpc/settings.yml index bdf2f6d52..56616dd59 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -60,7 +60,7 @@ wrapper_shell: /bin/bash # If you want a custom wrapper script for a one-off container, define it there with the script # stored alongside the container.yaml. wrapper_scripts: - enabled: false + enabled: true # use for docker aliases (set to null to disable) docker: docker.sh From 10a4289183fc391bbc707462a2e24484c9be4f8b Mon Sep 17 00:00:00 2001 From: vsoch Date: Sun, 20 Feb 2022 21:00:21 -0700 Subject: [PATCH 31/38] dont enable! Signed-off-by: vsoch --- shpc/settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shpc/settings.yml b/shpc/settings.yml index 56616dd59..bdf2f6d52 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -60,7 +60,7 @@ wrapper_shell: /bin/bash # If you want a custom wrapper script for a one-off container, define it there with the script # stored alongside the container.yaml. wrapper_scripts: - enabled: true + enabled: false # use for docker aliases (set to null to disable) docker: docker.sh From e73e16e7d2eb4e5addcbd78f49d37954b8e0ffd5 Mon Sep 17 00:00:00 2001 From: vsoch Date: Sun, 20 Feb 2022 22:32:29 -0700 Subject: [PATCH 32/38] handle parsing of boolean Signed-off-by: vsoch --- shpc/main/settings.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/shpc/main/settings.py b/shpc/main/settings.py index a19043acb..9698ddd72 100644 --- a/shpc/main/settings.py +++ b/shpc/main/settings.py @@ -145,6 +145,8 @@ def add(self, key, value): """ Add a value to a list parameter """ + value = self.parse_boolean(value) + # We can only add to lists current = self._settings.get(key) if current and not isinstance(current, list): @@ -176,12 +178,21 @@ def remove(self, key, value): "Warning: Check with shpc config edit - ordering of list can change." ) + def parse_boolean(self, value): + """ + If the value is True/False, ensure we return a boolean + """ + if isinstance(value, str) and value.lower() == "true": + value = True + elif isinstance(value, str) and value.lower() == "false": + value = False + return value + def set(self, key, value): """ Set a setting based on key and value. If the key has :, it's nested """ - value = True if value == "true" else value - value = False if value == "false" else value + value = self.parse_boolean(value) # List values not allowed for set current = self._settings.get(key) @@ -191,6 +202,7 @@ def set(self, key, value): # This is a reference to a dictionary (object) setting if isinstance(value, str) and ":" in value: subkey, value = value.split(":") + value = self.parse_boolean(value) self._settings[key][subkey] = value else: self._settings[key] = value From e54e93c2636ef24f2364bf929cbdb6fccd006c23 Mon Sep 17 00:00:00 2001 From: vsoch Date: Mon, 21 Feb 2022 00:17:43 -0700 Subject: [PATCH 33/38] fixing parsing of null issues Signed-off-by: vsoch --- shpc/main/modules/templates/singularity.lua | 4 ++-- shpc/main/schemas.py | 6 +++--- shpc/main/settings.py | 13 +++++++++++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index 254832034..5750bc062 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -18,7 +18,7 @@ Commands include: - {|module_name|}-run: singularity run {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} "$@" - {|module_name|}-shell: - singularity shell -s {{ singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} + singularity shell -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} - {|module_name|}-exec: singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} "$@" - {|module_name|}-inspect-runscript: @@ -50,7 +50,7 @@ if not os.getenv("SINGULARITY_COMMAND_OPTS") then setenv ("SINGULARITY_COMMAND_O -- interactive shell to any container, plus exec for aliases local containerPath = '{{ container_sif }}' -local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath +local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath local inspectCmd = "singularity ${SINGULARITY_OPTS} inspect ${SINGULARITY_COMMAND_OPTS} " diff --git a/shpc/main/schemas.py b/shpc/main/schemas.py index d458d2829..ed7f17798 100644 --- a/shpc/main/schemas.py +++ b/shpc/main/schemas.py @@ -112,9 +112,9 @@ "type": "object", "properties": { "enabled": {"type": "boolean"}, - "docker": {"type": "string"}, - "podman": {"type": "string"}, - "singularity": {"type": "string"}, + "docker": {"type": ["string", "null"]}, + "podman": {"type": ["string", "null"]}, + "singularity": {"type": ["string", "null"]}, }, } diff --git a/shpc/main/settings.py b/shpc/main/settings.py index 9698ddd72..8f8edacf3 100644 --- a/shpc/main/settings.py +++ b/shpc/main/settings.py @@ -151,6 +151,7 @@ def add(self, key, value): current = self._settings.get(key) if current and not isinstance(current, list): logger.exit("You cannot only add to a list variable.") + value = self.parse_null(value) if value not in current: # Add to the beginning of the list @@ -188,6 +189,14 @@ def parse_boolean(self, value): value = False return value + def parse_null(self, value): + """ + Given a null or none from the command line, ensure parsed as None type + """ + if isinstance(value, str) and value.lower() in ["none", "null"]: + return None + return value + def set(self, key, value): """ Set a setting based on key and value. If the key has :, it's nested @@ -203,8 +212,10 @@ def set(self, key, value): if isinstance(value, str) and ":" in value: subkey, value = value.split(":") value = self.parse_boolean(value) + value = self.parse_null(value) self._settings[key][subkey] = value else: + value = self.parse_null(value) self._settings[key] = value # Validate and catch error message cleanly @@ -233,6 +244,8 @@ def _substitutions(self, value): elif isinstance(value, dict): return value + if value.lower() in ["null", "None"]: + return None for rep, repvalue in defaults.reps.items(): if isinstance(value, list): value = [x.replace(rep, repvalue) for x in value] From 899ed28e239b85e5043561468525904ce6581f5c Mon Sep 17 00:00:00 2001 From: vsoch Date: Mon, 21 Feb 2022 00:20:38 -0700 Subject: [PATCH 34/38] bug Signed-off-by: vsoch --- shpc/main/settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/shpc/main/settings.py b/shpc/main/settings.py index 8f8edacf3..0bd700d15 100644 --- a/shpc/main/settings.py +++ b/shpc/main/settings.py @@ -244,8 +244,6 @@ def _substitutions(self, value): elif isinstance(value, dict): return value - if value.lower() in ["null", "None"]: - return None for rep, repvalue in defaults.reps.items(): if isinstance(value, list): value = [x.replace(rep, repvalue) for x in value] From b34280fd454df10ba0d4026368e14c1a08ba18dc Mon Sep 17 00:00:00 2001 From: vsoch Date: Tue, 22 Feb 2022 21:22:33 -0700 Subject: [PATCH 35/38] remove wrapper subdir we can literally just hard code bin, it is easier that way! Signed-off-by: vsoch --- shpc/main/modules/__init__.py | 1 - shpc/main/modules/templates/docker.lua | 2 +- shpc/main/modules/templates/docker.tcl | 2 +- shpc/main/modules/templates/docs.md | 2 +- shpc/main/modules/templates/singularity.lua | 2 +- shpc/main/modules/templates/singularity.tcl | 2 +- shpc/main/schemas.py | 1 - shpc/main/wrappers/base.py | 4 ++-- shpc/settings.yml | 3 --- 9 files changed, 7 insertions(+), 12 deletions(-) diff --git a/shpc/main/modules/__init__.py b/shpc/main/modules/__init__.py index b580972c6..c07886ee2 100644 --- a/shpc/main/modules/__init__.py +++ b/shpc/main/modules/__init__.py @@ -192,7 +192,6 @@ def docgen(self, module_name, out=None): versions=config.tags.keys(), github_url=github_url, container_url=config.url, - prefix=self.settings.module_exc_prefix, creation_date=datetime.now(), name=module_name, flatname=module_name.replace(os.sep, "-"), diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua index 145a638f1..1532fce61 100644 --- a/shpc/main/modules/templates/docker.lua +++ b/shpc/main/modules/templates/docker.lua @@ -59,7 +59,7 @@ set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) -- if we have any wrapper scripts, add the bin directory -{% if wrapper_scripts %}prepend_path("PATH", pathJoin(myFileName():match("(.*[/])") or ".", "{{ settings.wrapper_subdir }}")){% endif %} +{% if wrapper_scripts %}prepend_path("PATH", pathJoin(myFileName():match("(.*[/])") or ".", "bin")){% endif %} -- "aliases" to module commands - generate only if not a wrapper script already generated {% if aliases %}{% for alias in aliases %}{% if alias.name not in wrapper_scripts %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }} \"$@\"", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }}"){% endif %} diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl index 40e58419d..a926c7bfb 100644 --- a/shpc/main/modules/templates/docker.tcl +++ b/shpc/main/modules/templates/docker.tcl @@ -74,7 +74,7 @@ set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}" set-alias {|module_name|}-shell "${shellCmd}" # wrapper scripts? Add bin to path -{% if wrapper_scripts %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/{{ settings.wrapper_subdir }}"{% endif %} +{% if wrapper_scripts %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/bin"{% endif %} # "aliases" to module commands {% if aliases %}if { [ module-info shell bash ] } { diff --git a/shpc/main/modules/templates/docs.md b/shpc/main/modules/templates/docs.md index 2eb7579f0..35eb75e0d 100644 --- a/shpc/main/modules/templates/docs.md +++ b/shpc/main/modules/templates/docs.md @@ -58,7 +58,7 @@ $ podman run --rm {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endi $ docker run --rm {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} ``` -#### {|module_name|}-settings.singularity_shell: +#### {|module_name|}-shell: ```bash $ singularity settings.singularity_shell -s {{ settings.singularity_shell }} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index 5750bc062..ca6d30f68 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -62,7 +62,7 @@ set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) -- if we have any wrapper scripts, add bin to path -{% if wrapper_scripts %}prepend_path("PATH",pathJoin(myFileName():match("(.*[/])") or ".", "{{ settings.wrapper_subdir }}")){% endif %} +{% if wrapper_scripts %}prepend_path("PATH",pathJoin(myFileName():match("(.*[/])") or ".", "bin")){% endif %} -- "aliases" to module commands {% if aliases %}{% for alias in aliases %}{% if alias.name not in wrapper_scripts %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }} \"$@\"", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }}"){% endif %} diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl index 4484821fd..51ec418b4 100644 --- a/shpc/main/modules/templates/singularity.tcl +++ b/shpc/main/modules/templates/singularity.tcl @@ -80,7 +80,7 @@ set-alias {|module_name|}-shell "${shellCmd}" # if we have any wrapper scripts, add bin to path -{% if wrapper_scripts %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/{{ settings.wrapper_subdir }}"{% endif %} +{% if wrapper_scripts %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/bin"{% endif %} # "aliases" to module commands {% if aliases %}if { [ module-info shell bash ] } { diff --git a/shpc/main/schemas.py b/shpc/main/schemas.py index ed7f17798..dc68c32b6 100644 --- a/shpc/main/schemas.py +++ b/shpc/main/schemas.py @@ -139,7 +139,6 @@ "default_version": {"type": "boolean"}, "enable_tty": {"type": "boolean"}, "wrapper_scripts": wrapper_scripts, - "wrapper_subdir": {"type": "string"}, "container_tech": {"type": "string", "enum": ["singularity", "podman", "docker"]}, "singularity_shell": {"type": "string", "enum": shells}, "podman_shell": {"type": "string", "enum": shells}, diff --git a/shpc/main/wrappers/base.py b/shpc/main/wrappers/base.py index 847a47746..3ece90346 100644 --- a/shpc/main/wrappers/base.py +++ b/shpc/main/wrappers/base.py @@ -96,7 +96,7 @@ def generate_aliases(self): self.load_template() # Write scripts into container directory - wrapper_dir = os.path.join(self.module_dir, self.settings.wrapper_subdir) + wrapper_dir = os.path.join(self.module_dir, "bin") shpc.utils.mkdirp([wrapper_dir]) generated = [] @@ -133,7 +133,7 @@ def generate(self, alias): self.load_template() # Write scripts into container directory - wrapper_dir = os.path.join(self.module_dir, self.settings.wrapper_subdir) + wrapper_dir = os.path.join(self.module_dir, "bin") shpc.utils.mkdirp([wrapper_dir]) wrapper_path = os.path.join(wrapper_dir, alias) return [self._generate(wrapper_path, alias)] diff --git a/shpc/settings.yml b/shpc/settings.yml index bdf2f6d52..36be210c7 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -71,9 +71,6 @@ wrapper_scripts: # use for singularity aliases (set to null to disable) singularity: singularity.sh -# Wrapper scripts are stored in this subdirectory of the module directory -wrapper_subdir: "bin" - # the module namespace you want to install from. E.g., if you have ghcr.io/autamus/clingo # and you set the namespace to ghcr.io/autamus, you can just do: shpc install clingo. namespace: From 9a8bd2057d53d3653b1d83f8ad813ea8fed2085a Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Thu, 24 Feb 2022 11:11:39 +0800 Subject: [PATCH 36/38] Add/wrapper scripts - minor polishing (#497) * fixed typos in module/templates/docs.md * minor polishing in settings.yml, wrappers_subdir * docs: edits to user and developer guides, for wrapper scripts * reverted a couple of shell occurrences (docker/podman) in modules/templates/docs.md * one more edit for shell in modules/templates/docs.md * Update docs/getting_started/user-guide.rst Co-authored-by: Vanessasaurus <814322+vsoch@users.noreply.github.com> --- docs/getting_started/developer-guide.rst | 15 ++++++++------- docs/getting_started/user-guide.rst | 23 +++++++++++++++++++---- shpc/main/modules/templates/docs.md | 12 ++++++------ 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/docs/getting_started/developer-guide.rst b/docs/getting_started/developer-guide.rst index b67131dbe..c99e873ab 100644 --- a/docs/getting_started/developer-guide.rst +++ b/docs/getting_started/developer-guide.rst @@ -203,7 +203,7 @@ Wrapper Script Singularity HPC allows exposure of two kinds of wrapper scripts: -1. A global level wrapper intended to replace aliases. E.g., if an alias "samtools" is typically a direct container call, enabling a wrapper will generate an executable script "samtools" in a bin associated with the container, added to the path, to call instead. This is desired for workflow tools that want to run scripts. This global script is defined in settings.yml and described in the user guide. +1. A global level wrapper intended to replace aliases. E.g., if an alias "samtools" is typically a direct container call, enabling a wrapper will generate an executable script "samtools" in a "bin" directory associated with the container, added to the path, to call instead. This is desired when MPI ("mpirun") or scheduler (e.g. "srun" with Slurm) utilities are needed to run the scripts. This global script is defined in settings.yml and described in the user guide. 2. A container level wrapper that is specific to a container, described here. For container specific scripts, you can add sections to a ``container.yaml`` to specify the script (and container type) @@ -233,13 +233,14 @@ the custom container script "fork": .. code-block:: console - $ tree containers/vanessa/salad/ - containers/vanessa/salad/ + $ tree modules/vanessa/salad/ + modules/vanessa/salad/ └── latest - ├── bin - │   ├── fork - │   └── salad - └── vanessa-salad-latest-sha256:e8302da47e3200915c1d3a9406d9446f04da7244e4995b7135afd2b79d4f63db.sif + ├── 99-shpc.sh + ├── bin + │   ├── fork + │   └── salad + └── module.lua If we disable all wrapper scripts, the bin directory would not exist. If we set the default wrapper scripts for singularity and docker in settings.yml and left enable to true, we would only see "fork." diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst index d2e5b4c45..304c2a927 100644 --- a/docs/getting_started/user-guide.rst +++ b/docs/getting_started/user-guide.rst @@ -201,6 +201,21 @@ variable replacement. A summary table of variables is included below, and then f * - test_shell - The shell used for the test.sh file - /bin/bash + * - wrapper_shell + - The shell used for wrapper scripts + - /bin/bash + * - wrapper_scripts:enabled + - enable or disable generation of wrapper scripts, instead of module aliases + - false + * - wrapper_scripts:docker + - The name of the generic wrapper script template for docker + - docker.sh + * - wrapper_scripts:podman + - The name of the generic wrapper script template for podman + - docker.sh + * - wrapper_scripts:singularity + - The name of the generic wrapper script template for singularity + - singularity.sh * - namespace - Set a default module namespace that you want to install from. - null @@ -451,7 +466,7 @@ Wrapper Scripts Singularity HPC allows for global definition of wrapper scripts, meaning that instead of writing a module alias to run a container for some given alias, we generate a wrapper script of the same name instead. Since the settings.yml is global, all wrapper scripts defined here are specific to replacing aliases. -Container-specific scripts you'll want to include in the container.yaml, described in the developer docs. Let's take a look at the settings: +Container-specific scripts you'll want to include in the container.yaml are described in the developer docs. Let's take a look at the settings: .. code-block:: yaml @@ -474,14 +489,14 @@ Container-specific scripts you'll want to include in the container.yaml, describ Since different container technologies might expose different environment variables (e.g., ``SINGULARITY_OPTS`` vs ``PODMAN_OPTS``) they are organized above based on the container technology. If you want to customize the wrapper script, simply replace the relative paths -above (e.g., ``singularity.sh`` with an absolute path to a file that will be used instead. For global alias scripts such as these, +above (e.g., ``singularity.sh``) with an absolute path to a file that will be used instead. For global alias scripts such as these, Singularity HPC will look for: 1. An absolute path first, if found is used first. 2. Then a script name in the shpc/main/wrappers directory Here is an example of using wrapper scripts for the "python" container, which doesn't have container specific wrappers. What you see -is the one entrypoint, "python" being placed in a bin that the module will see instead of defining the alias. +is the one entrypoint, `python`, being placed in a "bin" subdirectory that the module will see instead of defining the alias. .. code-block:: console @@ -533,7 +548,7 @@ Where are wrapper scripts stored? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Since we don't allow overlap -of the name of an alias wrapper script (e.g., ``bin/python``) as a wrapper to a python entrypoint) from a custom container wrapper script (e.g., a wrapper script with name "python" under a container.yaml) we can keep them both in the modules directory. If you see a need to put them elsewhere please let us kno. +of the name of an alias wrapper script (e.g., ``bin/python`` as a wrapper to a python entrypoint) from a custom container wrapper script (e.g., a wrapper script with name "python" under a container.yaml) we can keep them both in the modules directory. If you see a need to put them elsewhere please let us know. .. _getting_started-commands: diff --git a/shpc/main/modules/templates/docs.md b/shpc/main/modules/templates/docs.md index 35eb75e0d..1a30cfb20 100644 --- a/shpc/main/modules/templates/docs.md +++ b/shpc/main/modules/templates/docs.md @@ -47,7 +47,7 @@ You can use tab for auto-completion of module names or commands that are provide ### Commands -When you install this module, you'll be able to load it to make the following commands accessible. +When you install this module, you will be able to load it to make the following commands accessible. Examples for both Singularity, Podman, and Docker (container technologies supported) are included. #### {|module_name|}-run: @@ -61,15 +61,15 @@ $ docker run --rm {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endi #### {|module_name|}-shell: ```bash -$ singularity settings.singularity_shell -s {{ settings.singularity_shell }} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} -$ podman run --it --rm --entrypoint {{ settings.singularity_shell }} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} -$ docker run --it --rm --entrypoint {{ settings.singularity_shell }} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} +$ singularity shell -s {{ settings.singularity_shell }} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} +$ podman run --it --rm --entrypoint {{ settings.podman_shell }} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} +$ docker run --it --rm --entrypoint {{ settings.docker_shell }} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} ``` #### {|module_name|}-exec: ```bash -$ singularity exec -s {{ settings.singularity_shell }} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} "$@" +$ singularity exec {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} "$@" $ podman run --it --rm --entrypoint "" {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} "$@" $ docker run --it --rm --entrypoint "" {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %} -v ${PWD} -w ${PWD} "$@" ``` @@ -118,7 +118,7 @@ $ docker run --rm {% if settings.bindpaths %}-v {{ settings.bindpaths }}{% endif In the above, the `` directive will reference an actual container provided by the module, for the version you have chosen to load. An environment file in the module folder will also be bound. Note that although a container -might provide custom commands, every container exposes unique exec, settings.singularity_shell, run, and +might provide custom commands, every container exposes unique exec, shell, run, and inspect aliases. For anycommands above, you can export: - SINGULARITY_OPTS: to define custom options for singularity (e.g., --debug) From 4c7ac07a627af3a7a4772f95f2b8101baa452b8a Mon Sep 17 00:00:00 2001 From: vsoch Date: Wed, 23 Feb 2022 21:40:49 -0700 Subject: [PATCH 37/38] adding new design for templates: bases and snippets! Signed-off-by: vsoch --- docs/getting_started/developer-guide.rst | 69 ++++++++++++++++++- registry/vanessa/salad/docker_fork.sh | 6 +- registry/vanessa/salad/singularity_fork.sh | 6 +- shpc/main/schemas.py | 1 + shpc/main/wrappers/base.py | 25 +++++-- shpc/main/wrappers/docker.sh | 3 - shpc/main/wrappers/singularity.sh | 3 - shpc/main/wrappers/templates/__init__.py | 0 .../main/wrappers/templates/bases/__init__.py | 0 .../templates/bases/shell-script-base.sh | 7 ++ shpc/main/wrappers/templates/docker.sh | 4 ++ shpc/main/wrappers/templates/singularity.sh | 4 ++ .../wrappers/templates/snippets/__init__.py | 0 shpc/settings.yml | 3 + 14 files changed, 113 insertions(+), 18 deletions(-) delete mode 100755 shpc/main/wrappers/docker.sh delete mode 100755 shpc/main/wrappers/singularity.sh create mode 100644 shpc/main/wrappers/templates/__init__.py create mode 100644 shpc/main/wrappers/templates/bases/__init__.py create mode 100755 shpc/main/wrappers/templates/bases/shell-script-base.sh create mode 100755 shpc/main/wrappers/templates/docker.sh create mode 100755 shpc/main/wrappers/templates/singularity.sh create mode 100644 shpc/main/wrappers/templates/snippets/__init__.py diff --git a/docs/getting_started/developer-guide.rst b/docs/getting_started/developer-guide.rst index c99e873ab..f865fd1db 100644 --- a/docs/getting_started/developer-guide.rst +++ b/docs/getting_started/developer-guide.rst @@ -248,16 +248,15 @@ scripts for singularity and docker in settings.yml and left enable to true, we w How to write an alias wrapper script ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -First, decide if you want a global script (to replace or wrap aliases) OR a custom container script. For an alias derived script, you should: +First, decide if you want a global script (to replace or wrap aliases) OR a custom container script. For an alias derived (global) script, you should: 1. Write the new script file into shpc/main/wrappers. 2. Add an entry to shpc/main/wrappers/scripts referencing the script. -For global scripts, the user can select to use it in their settings.yaml. +For these global scripts, the user can select to use it in their settings.yaml. We will eventually write a command to list global wrappers available, so if you add a new one future users will know about it. For alias wrapper scripts, the following variables are passed for rendering: - .. list-table:: Title :widths: 15 15 40 30 :header-rows: 1 @@ -345,6 +344,70 @@ The following variables are passed for rendering. - ``{{ features.gpu }}`` +Templating for both wrapper script types +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Note that you are free to use "snippets" and "bases" either as an inclusion or "extends" meaning you can +easily re-use code. For example, if we have the following registered directories under ``shpc/main/wrappers/templates`` +for definition of bases and templates: + +.. code-block:: console + + main/wrappers/templates/ + + # These are intended for use with "extends" + ├── bases + │   ├── __init__.py + │   └── shell-script-base.sh + + # These are top level template files, as specified in the settings.yml + ├── docker.sh + ├── singularity.sh + + # A mostly empty directory ready for any snippets! + └── snippets + +For example, a "bases" template to define a shell and some special command that might look like this: + +.. code-block:: console + + #!{{ settings.wrapper_shell }} + + script=`realpath $0` + wrapper_bin=`dirname $script` + {% if '/csh' in settings.wrapper_shell %}set moduleDir=`dirname $wrapper_bin`{% else %}export moduleDir=$(dirname $wrapper_bin){% endif %} + + {% block content %}{% endblock %} + + +And then to use it for any container- or global- wrapper we would do the following in the wrapper script: + +.. code-block:: console + + {% extends "bases/my-base-shell.sh" %} + + # some custom wrapper before stuff here + + {% block content %}{% endblock %} + + # some custom wrapper after stuff here + + +For snippets, which are intended to be more chunks of code you can throw in one spot +on the fly, you can do this: + + +.. code-block:: console + + {% include "snippets/export-envars.sh" %} + # some custom wrapper after stuff here + + +Finally, if you want to add your own custom templates directory for which you +can refer to templates relatively, define ``wrapper_scripts`` -> ``templates`` as a full path +in your settings. + + Environment Variables --------------------- diff --git a/registry/vanessa/salad/docker_fork.sh b/registry/vanessa/salad/docker_fork.sh index ebee68583..cf1dca6f9 100755 --- a/registry/vanessa/salad/docker_fork.sh +++ b/registry/vanessa/salad/docker_fork.sh @@ -1,7 +1,9 @@ -#!{{ settings.wrapper_shell }} +{% extends "bases/shell-script-base.sh" %} + +# See shpc/main/wrappers/snippets for what is added via this extend # This is an example of a custom wrapper script for a docker container # Note that we are showing an example of all the default args provided to the script! # You don't need to use any of this syntax -{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} --entrypoint /code/salad {{ image }} fork {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% block content %}{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} --entrypoint /code/salad {{ image }} fork {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %}{% endblock %} diff --git a/registry/vanessa/salad/singularity_fork.sh b/registry/vanessa/salad/singularity_fork.sh index f6821f26a..1fd760259 100755 --- a/registry/vanessa/salad/singularity_fork.sh +++ b/registry/vanessa/salad/singularity_fork.sh @@ -1,7 +1,9 @@ -#!{{ settings.wrapper_shell }} +{% extends "bases/shell-script-base.sh" %} + +# See shpc/main/wrappers/snippets for what is added via this extend # This is an example of a singularity wrapper script, under singularity_scripts # Note that we are showing an example of all the default args provided to the script! # You don't need to use any of this syntax -singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environent_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {{ image }} /code/salad fork {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% block content %}singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environent_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {{ image }} /code/salad fork {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %}{% endblock %} diff --git a/shpc/main/schemas.py b/shpc/main/schemas.py index dc68c32b6..a2beaf393 100644 --- a/shpc/main/schemas.py +++ b/shpc/main/schemas.py @@ -114,6 +114,7 @@ "enabled": {"type": "boolean"}, "docker": {"type": ["string", "null"]}, "podman": {"type": ["string", "null"]}, + "templates": {"type": ["string", "null"]}, "singularity": {"type": ["string", "null"]}, }, } diff --git a/shpc/main/wrappers/base.py b/shpc/main/wrappers/base.py index 3ece90346..ec52d2775 100644 --- a/shpc/main/wrappers/base.py +++ b/shpc/main/wrappers/base.py @@ -3,11 +3,12 @@ __license__ = "MPL 2.0" from shpc.logger import logger -from jinja2 import Template +from jinja2 import Template, Environment, FileSystemLoader import shpc.utils import os here = os.path.dirname(os.path.abspath(__file__)) +templates = os.path.join(here, "templates") class WrapperScript: @@ -76,7 +77,7 @@ def load_template(self): # Finally, it has to be in the template directory if not template_file: - template_file = os.path.join(here, self.wrapper_template) + template_file = os.path.join(templates, self.wrapper_template) if not os.path.exists(template_file): logger.exit( @@ -84,10 +85,24 @@ def load_template(self): % self.wrapper_template ) - with open(template_file, "r") as temp: - template = Template(temp.read()) self.template_file = template_file - self.template = template + + # Default to search here in template file path, then in templates + template_paths = [os.path.dirname(template_file), templates] + + # Do we have a custom template path directory? + if "templates" in self.settings.wrapper_scripts: + path = self.settings.wrapper_scripts["templates"] + if not os.path.exists(path): + logger.exit( + "%s designated as a templates directory, but it does not exist." + % path + ) + template_paths = [path] + template_paths + + loader = FileSystemLoader(template_paths) + env = Environment(loader=loader) + self.template = env.get_template(os.path.basename(template_file)) def generate_aliases(self): """ diff --git a/shpc/main/wrappers/docker.sh b/shpc/main/wrappers/docker.sh deleted file mode 100755 index 7afbdb1ba..000000000 --- a/shpc/main/wrappers/docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!{{ settings.wrapper_shell }} - -{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} --entrypoint {{ alias.entrypoint }} {{ image }} {{ alias.args }} {% if '/sh' in wrapper_shell or '/bash' in wrapper_shell %}"$@"{% elif '/csh' in wrapper_shell %}$argv:q{% endif %} diff --git a/shpc/main/wrappers/singularity.sh b/shpc/main/wrappers/singularity.sh deleted file mode 100755 index baedb5deb..000000000 --- a/shpc/main/wrappers/singularity.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!{{ settings.wrapper_shell }} - -singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {% if alias.singularity_options %} {{ alias.singularity_options }} {% endif %} {{ image }} {{ alias.command }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} diff --git a/shpc/main/wrappers/templates/__init__.py b/shpc/main/wrappers/templates/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/shpc/main/wrappers/templates/bases/__init__.py b/shpc/main/wrappers/templates/bases/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/shpc/main/wrappers/templates/bases/shell-script-base.sh b/shpc/main/wrappers/templates/bases/shell-script-base.sh new file mode 100755 index 000000000..f6172e6bb --- /dev/null +++ b/shpc/main/wrappers/templates/bases/shell-script-base.sh @@ -0,0 +1,7 @@ +#!{{ settings.wrapper_shell }} + +script=`realpath $0` +wrapper_bin=`dirname $script` +{% if '/csh' in settings.wrapper_shell %}set moduleDir=`dirname $wrapper_bin`{% else %}export moduleDir=$(dirname $wrapper_bin){% endif %} + +{% block content %}{% endblock %} diff --git a/shpc/main/wrappers/templates/docker.sh b/shpc/main/wrappers/templates/docker.sh new file mode 100755 index 000000000..6d40fd198 --- /dev/null +++ b/shpc/main/wrappers/templates/docker.sh @@ -0,0 +1,4 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} --entrypoint {{ alias.entrypoint }} {{ image }} {{ alias.args }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% endblock %} diff --git a/shpc/main/wrappers/templates/singularity.sh b/shpc/main/wrappers/templates/singularity.sh new file mode 100755 index 000000000..c187401df --- /dev/null +++ b/shpc/main/wrappers/templates/singularity.sh @@ -0,0 +1,4 @@ +{% extends "bases/shell-script-base.sh" %} + +{% block content %}singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {% if alias.singularity_options %} {{ alias.singularity_options }} {% endif %} {{ image }} {{ alias.command }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% endblock %} diff --git a/shpc/main/wrappers/templates/snippets/__init__.py b/shpc/main/wrappers/templates/snippets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/shpc/settings.yml b/shpc/settings.yml index 36be210c7..b02344346 100644 --- a/shpc/settings.yml +++ b/shpc/settings.yml @@ -71,6 +71,9 @@ wrapper_scripts: # use for singularity aliases (set to null to disable) singularity: singularity.sh + # Add an extra custom template directory (searched first) + templates: null + # the module namespace you want to install from. E.g., if you have ghcr.io/autamus/clingo # and you set the namespace to ghcr.io/autamus, you can just do: shpc install clingo. namespace: From 6f8c0c70fd4733387bfac3796359aed7a9816dfa Mon Sep 17 00:00:00 2001 From: Marco De La Pierre Date: Thu, 3 Mar 2022 00:38:34 +0800 Subject: [PATCH 38/38] Add/wrapper scripts (#499) * wrapper/templates/bases: csh always needs "set" for variables * wrappers/templates: using variable moduleDir * modules/templates lua: moduleDir always replaces {{ module_dir }} * modules/templates tcl: moduleDir always replaces {{ module_dir }} * main/container: no more need to pass module_dir to template.render for modulefiles * registry/vanessa/salad: updated wrapper scripts with moduleDir var * registry/vanessa/salad: typo in singularity_fork.sh --- registry/vanessa/salad/docker_fork.sh | 2 +- registry/vanessa/salad/singularity_fork.sh | 2 +- shpc/main/container/docker.py | 1 - shpc/main/container/singularity.py | 1 - shpc/main/modules/templates/docker.lua | 20 +++++++++---------- shpc/main/modules/templates/docker.tcl | 20 ++++++++++--------- shpc/main/modules/templates/singularity.lua | 20 +++++++++---------- shpc/main/modules/templates/singularity.tcl | 20 ++++++++++--------- .../templates/bases/shell-script-base.sh | 6 +++--- shpc/main/wrappers/templates/docker.sh | 2 +- shpc/main/wrappers/templates/singularity.sh | 2 +- 11 files changed, 49 insertions(+), 47 deletions(-) diff --git a/registry/vanessa/salad/docker_fork.sh b/registry/vanessa/salad/docker_fork.sh index cf1dca6f9..a122e8f7b 100755 --- a/registry/vanessa/salad/docker_fork.sh +++ b/registry/vanessa/salad/docker_fork.sh @@ -6,4 +6,4 @@ # Note that we are showing an example of all the default args provided to the script! # You don't need to use any of this syntax -{% block content %}{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} --entrypoint /code/salad {{ image }} fork {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %}{% endblock %} +{% block content %}{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file $moduleDir/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} --entrypoint /code/salad {{ image }} fork {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %}{% endblock %} diff --git a/registry/vanessa/salad/singularity_fork.sh b/registry/vanessa/salad/singularity_fork.sh index 1fd760259..e9a98b80a 100755 --- a/registry/vanessa/salad/singularity_fork.sh +++ b/registry/vanessa/salad/singularity_fork.sh @@ -6,4 +6,4 @@ # Note that we are showing an example of all the default args provided to the script! # You don't need to use any of this syntax -{% block content %}singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environent_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {{ image }} /code/salad fork {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %}{% endblock %} +{% block content %}singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B $moduleDir/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {{ image }} /code/salad fork {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %}{% endblock %} diff --git a/shpc/main/container/docker.py b/shpc/main/container/docker.py index 7a096def7..ed8c9a038 100644 --- a/shpc/main/container/docker.py +++ b/shpc/main/container/docker.py @@ -244,7 +244,6 @@ def install( shell=shell, image=container_path, description=description, - module_dir=module_dir, aliases=aliases, url=url, features=features, diff --git a/shpc/main/container/singularity.py b/shpc/main/container/singularity.py index b177fd5fc..dbe7b96cc 100644 --- a/shpc/main/container/singularity.py +++ b/shpc/main/container/singularity.py @@ -192,7 +192,6 @@ def install( url=url, features=features, version=version, - module_dir=module_dir, labels=labels, deffile=deffile, creation_date=datetime.now(), diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua index 1532fce61..780311678 100644 --- a/shpc/main/modules/templates/docker.lua +++ b/shpc/main/modules/templates/docker.lua @@ -16,16 +16,16 @@ Container: Commands include: - {|module_name|}-run: - {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}-v ${PWD} -w ${PWD} "$@" + {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file /{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}-v ${PWD} -w ${PWD} "$@" - {|module_name|}-shell: - {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}--entrypoint {{ shell }} -v ${PWD} -w ${PWD} + {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file /{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}--entrypoint {{ shell }} -v ${PWD} -w ${PWD} - {|module_name|}-exec: - {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint "" {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} "$@" + {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint "" {% if settings.environment_file %}--env-file /{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} "$@" - {|module_name|}-inspect: {{ command }} inspect {% if aliases %}{% for alias in aliases %} - {{ alias.name }}: - {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ alias.entrypoint }} {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options }} {% endif %} -v ${PWD} -w ${PWD} "{{ alias.args }}" "$@" + {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ alias.entrypoint }} {% if settings.environment_file %}--env-file /{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options }} {% endif %} -v ${PWD} -w ${PWD} "{{ alias.args }}" "$@" {% endfor %}{% endif %} For each of the above, you can export: @@ -40,16 +40,16 @@ For each of the above, you can export: if not os.getenv("PODMAN_OPTS") then setenv ("PODMAN_OPTS", "") end if not os.getenv("PODMAN_COMMAND_OPTS") then setenv ("PODMAN_COMMAND_OPTS", "") end --- we probably do not need this -local MODULEPATH="{{ module_dir }}" +-- directory containing this modulefile (dynamically defined) +local moduleDir = myFileName():match("(.*[/])") or "." -- interactive shell to any container, plus exec for aliases local containerPath = '{{ image }}' -local shellCmd = "{{ command }} ${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} ${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath +local shellCmd = "{{ command }} ${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} ${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath -- execCmd needs entrypoint to be the executor -local execCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " -local runCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath +local execCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " +local runCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath local inspectCmd = "{{ command }} ${PODMAN_OPTS} inspect ${PODMAN_COMMAND_OPTS} " .. containerPath -- set_shell_function takes bashStr and cshStr @@ -59,7 +59,7 @@ set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) -- if we have any wrapper scripts, add the bin directory -{% if wrapper_scripts %}prepend_path("PATH", pathJoin(myFileName():match("(.*[/])") or ".", "bin")){% endif %} +{% if wrapper_scripts %}prepend_path("PATH", pathJoin(moduleDir, "bin")){% endif %} -- "aliases" to module commands - generate only if not a wrapper script already generated {% if aliases %}{% for alias in aliases %}{% if alias.name not in wrapper_scripts %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }} \"$@\"", execCmd .. {% if alias.docker_options %} "{{ alias.docker_options }} " .. {% endif %} " --entrypoint {{ alias.entrypoint }} " .. containerPath .. " {{ alias.args }}"){% endif %} diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl index a926c7bfb..ade20ceb4 100644 --- a/shpc/main/modules/templates/docker.tcl +++ b/shpc/main/modules/templates/docker.tcl @@ -15,16 +15,16 @@ proc ModulesHelp { } { puts stderr " - {{ image }}" puts stderr "Commands include:" puts stderr " - {|module_name|}-run:" - puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . \"\$@\"" + puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file /{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . \"\$@\"" puts stderr " - {|module_name|}-shell:" - puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %} --env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . " + puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %} --env-file /{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . " puts stderr " - {|module_name|}-exec:" - puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\" {% if settings.environment_file %} --env-file {{ module_dir }}/{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . \"\$@\"" + puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\" {% if settings.environment_file %} --env-file /{{ settings.environment_file }} {% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v . -w . \"\$@\"" puts stderr " - {|module_name|}-inspect:" puts stderr " {{ command }} inspect " puts stderr "" {% if aliases %}{% for alias in aliases %} puts stderr " - {{ alias.name }}:" - puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} --rm -u `id -u`:`id -g` --entrypoint {{ alias.entrypoint | replace("$", "\$") }} {% if settings.environment_file %}--settings.environment_file {{ module_dir }}/{{ settings.environment_file }} {% endif %}{% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options | replace("$", "\$") }} {% endif %} -v . -w . {{ alias.args | replace("$", "\$") }} \"\$@\"" + puts stderr " {{ command }} run -i{% if settings.enable_tty %}t{% endif %} --rm -u `id -u`:`id -g` --entrypoint {{ alias.entrypoint | replace("$", "\$") }} {% if settings.environment_file %}--settings.environment_file /{{ settings.environment_file }} {% endif %}{% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %}{% if alias.docker_options %}{{ alias.docker_options | replace("$", "\$") }} {% endif %} -v . -w . {{ alias.args | replace("$", "\$") }} \"\$@\"" {% endfor %}{% endif %} puts stderr "" puts stderr "For each of the above, you can export:" @@ -46,7 +46,6 @@ if { ![info exists ::env(PODMAN_COMMAND_OPTS)] } { set name "{{ name }}" set version "{{ version }}" -set progdir "{{ module_dir }}" set description "$name - $version" set containerPath "{{ image }}" set workdir [pwd] @@ -56,6 +55,9 @@ set helpcommand "This module is a {{ docker }} container wrapper for {{ name }} {% if labels %}{% for key, value in labels.items() %}set {{ key }} "{{ value }}" {% endfor %}{% endif %} +# directory containing this modulefile (dynamically defined) +set moduleDir "[file dirname ${ModulesCurrentModulefile}]" + # conflict with modules with the same alias name conflict {{ parsed_name.tool }} {% if name != parsed_name.tool %}conflict {{ name }}{% endif %} @@ -63,18 +65,18 @@ conflict {{ parsed_name.tool }} {% endfor %}{% endif %} # interactive shell to any container, plus exec for aliases -set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if settings.enable_tty %}t{% endif %} --entrypoint {{ shell }} {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" +set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if settings.enable_tty %}t{% endif %} --entrypoint {{ shell }} {% if settings.environment_file %}--env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" # execCmd needs entrypoint to be the executor -set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if settings.environment_file %} --env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }}{% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir" -set runCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" +set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if settings.environment_file %} --env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }}{% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir" +set runCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}" # set_shell_function takes bashStr and cshStr set-alias {|module_name|}-shell "${shellCmd}" # wrapper scripts? Add bin to path -{% if wrapper_scripts %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/bin"{% endif %} +{% if wrapper_scripts %}prepend-path PATH "${moduleDir}/bin"{% endif %} # "aliases" to module commands {% if aliases %}if { [ module-info shell bash ] } { diff --git a/shpc/main/modules/templates/singularity.lua b/shpc/main/modules/templates/singularity.lua index ca6d30f68..f50c7ec4f 100644 --- a/shpc/main/modules/templates/singularity.lua +++ b/shpc/main/modules/templates/singularity.lua @@ -16,18 +16,18 @@ Container: Commands include: - {|module_name|}-run: - singularity run {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} "$@" + singularity run {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} "$@" - {|module_name|}-shell: - singularity shell -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} + singularity shell -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} - {|module_name|}-exec: - singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} "$@" + singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} "$@" - {|module_name|}-inspect-runscript: singularity inspect -r - {|module_name|}-inspect-deffile: singularity inspect -d {% if aliases %}{% for alias in aliases %} - {{ alias.name }}: - singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options }} {% endif %} {{ alias.command }} "$@" + singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options }} {% endif %} {{ alias.command }} "$@" {% endfor %}{% endif %} For each of the above, you can export: @@ -38,8 +38,8 @@ For each of the above, you can export: {% if settings.singularity_module %}load("{{ settings.singularity_module }}"){% endif %} --- we probably do not need this -local MODULEPATH="{{ module_dir }}" +-- directory containing this modulefile (dynamically defined) +local moduleDir = myFileName():match("(.*[/])") or "." -- singularity environment variable to set shell setenv("SINGULARITY_SHELL", "{{ settings.singularity_shell }}") @@ -50,9 +50,9 @@ if not os.getenv("SINGULARITY_COMMAND_OPTS") then setenv ("SINGULARITY_COMMAND_O -- interactive shell to any container, plus exec for aliases local containerPath = '{{ container_sif }}' -local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath -local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " -local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath +local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath +local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " +local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath local inspectCmd = "singularity ${SINGULARITY_OPTS} inspect ${SINGULARITY_COMMAND_OPTS} " -- set_shell_function takes bashStr and cshStr @@ -62,7 +62,7 @@ set_shell_function("{|module_name|}-shell", shellCmd, shellCmd) conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %}) -- if we have any wrapper scripts, add bin to path -{% if wrapper_scripts %}prepend_path("PATH",pathJoin(myFileName():match("(.*[/])") or ".", "bin")){% endif %} +{% if wrapper_scripts %}prepend_path("PATH", pathJoin(moduleDir, "bin")){% endif %} -- "aliases" to module commands {% if aliases %}{% for alias in aliases %}{% if alias.name not in wrapper_scripts %}set_shell_function("{{ alias.name }}", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }} \"$@\"", execCmd .. {% if alias.singularity_options %} "{{ alias.singularity_options }} " .. {% endif %} containerPath .. " {{ alias.command }}"){% endif %} diff --git a/shpc/main/modules/templates/singularity.tcl b/shpc/main/modules/templates/singularity.tcl index 51ec418b4..dad790d99 100644 --- a/shpc/main/modules/templates/singularity.tcl +++ b/shpc/main/modules/templates/singularity.tcl @@ -18,18 +18,18 @@ proc ModulesHelp { } { puts stderr "Commands include:" puts stderr "" puts stderr " - {|module_name|}-run:" - puts stderr " singularity run {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} \"\$@\"" + puts stderr " singularity run {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} \"\$@\"" puts stderr " - {|module_name|}-shell:" - puts stderr " singularity shell -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}" + puts stderr " singularity shell -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}" puts stderr " - {|module_name|}-exec:" - puts stderr " singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} \"\$@\"" + puts stderr " singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %} \"\$@\"" puts stderr " - {|module_name|}-inspect-runscript:" puts stderr " singularity inspect -r " puts stderr " - {|module_name|}-inspect-deffile:" puts stderr " singularity inspect -d " puts stderr "" {% if aliases %}{% for alias in aliases %} puts stderr " - {{ alias.name }}:" - puts stderr " singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options | replace("$", "\$") }} {% endif %} {{ alias.command | replace("$", "\$") }} \"\$@\"" + puts stderr " singularity exec {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B /{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }} {% endif %}{% if alias.singularity_options %}{{ alias.singularity_options | replace("$", "\$") }} {% endif %} {{ alias.command | replace("$", "\$") }} \"\$@\"" {% endfor %}{% endif %} puts stderr "" puts stderr "For each of the above, you can export:" @@ -51,7 +51,6 @@ if { ![info exists ::env(SINGULARITY_COMMAND_OPTS)] } { set name {{ name }} set version {{ version }} -set progdir {{ module_dir }} set description "$name - $version" set containerPath {{ container_sif }} {% if description %}set notes "{{ description }}"{% endif %} @@ -60,6 +59,9 @@ set helpcommand "This module is a singularity container wrapper for {{ name }} v {% if labels %}{% for key, value in labels.items() %}set {{ key }} "{{ value }}" {% endfor %}{% endif %} +# directory containing this modulefile (dynamically defined) +set moduleDir "[file dirname ${ModulesCurrentModulefile}]" + # conflict with modules with the same alias name conflict {{ parsed_name.tool }} {% if name != parsed_name.tool %}conflict {{ name }}{% endif %} @@ -70,9 +72,9 @@ conflict {{ parsed_name.tool }} setenv SINGULARITY_SHELL {{ settings.singularity_shell }} # interactive shell to any container, plus exec for aliases -set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}" -set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " -set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}" +set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}" +set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " +set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}" set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_OPTS} " # set_shell_function takes bashStr and cshStr @@ -80,7 +82,7 @@ set-alias {|module_name|}-shell "${shellCmd}" # if we have any wrapper scripts, add bin to path -{% if wrapper_scripts %}prepend-path PATH "[file dirname ${ModulesCurrentModulefile}]/bin"{% endif %} +{% if wrapper_scripts %}prepend-path PATH "${moduleDir}/bin"{% endif %} # "aliases" to module commands {% if aliases %}if { [ module-info shell bash ] } { diff --git a/shpc/main/wrappers/templates/bases/shell-script-base.sh b/shpc/main/wrappers/templates/bases/shell-script-base.sh index f6172e6bb..111f736c4 100755 --- a/shpc/main/wrappers/templates/bases/shell-script-base.sh +++ b/shpc/main/wrappers/templates/bases/shell-script-base.sh @@ -1,7 +1,7 @@ #!{{ settings.wrapper_shell }} -script=`realpath $0` -wrapper_bin=`dirname $script` -{% if '/csh' in settings.wrapper_shell %}set moduleDir=`dirname $wrapper_bin`{% else %}export moduleDir=$(dirname $wrapper_bin){% endif %} +{% if '/csh' in settings.wrapper_shell %}set {% endif %}script=`realpath $0` +{% if '/csh' in settings.wrapper_shell %}set {% endif %}wrapper_bin=`dirname $script` +{% if '/csh' in settings.wrapper_shell %}set {% endif %}moduleDir=`dirname $wrapper_bin` {% block content %}{% endblock %} diff --git a/shpc/main/wrappers/templates/docker.sh b/shpc/main/wrappers/templates/docker.sh index 6d40fd198..c9d8dfd6c 100755 --- a/shpc/main/wrappers/templates/docker.sh +++ b/shpc/main/wrappers/templates/docker.sh @@ -1,4 +1,4 @@ {% extends "bases/shell-script-base.sh" %} -{% block content %}{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file {{ module_dir }}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} --entrypoint {{ alias.entrypoint }} {{ image }} {{ alias.args }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% block content %}{{ container.command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file $moduleDir/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} {% if alias.docker_options %} {{ alias.docker_options }} {% endif %} --entrypoint {{ alias.entrypoint }} {{ image }} {{ alias.args }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} {% endblock %} diff --git a/shpc/main/wrappers/templates/singularity.sh b/shpc/main/wrappers/templates/singularity.sh index c187401df..98ab377c8 100755 --- a/shpc/main/wrappers/templates/singularity.sh +++ b/shpc/main/wrappers/templates/singularity.sh @@ -1,4 +1,4 @@ {% extends "bases/shell-script-base.sh" %} -{% block content %}singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B {{ module_dir }}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {% if alias.singularity_options %} {{ alias.singularity_options }} {% endif %} {{ image }} {{ alias.command }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} +{% block content %}singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B $moduleDir/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} {% if alias.singularity_options %} {{ alias.singularity_options }} {% endif %} {{ image }} {{ alias.command }} {% if '/sh' in settings.wrapper_shell or '/bash' in settings.wrapper_shell %}"$@"{% elif '/csh' in settings.wrapper_shell %}$argv:q{% endif %} {% endblock %}