diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f95d5671..e4bc39239 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,9 +31,9 @@ update_go: &update_go name: Update Go to 1.14.9 working_directory: /tmp command: |- - wget https://dl.google.com/go/go1.14.9.linux-amd64.tar.gz + wget https://dl.google.com/go/go1.16.5.linux-amd64.tar.gz sudo rm -rf $GOROOT - sudo tar -C /usr/local -xzf go1.14.9.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go1.16.5.linux-amd64.tar.gz sudo ln -s $GOROOT/bin/go /usr/local/bin/go fetch_deb_deps: &fetch_deb_deps @@ -50,9 +50,9 @@ build_singularity: &install_singularity working_directory: ~/go/singularity command: |- cd ~/go - wget https://github.com/hpcng/singularity/releases/download/v3.7.1/singularity-3.7.1.tar.gz && \ - tar -xzvf singularity-3.7.1.tar.gz && \ - cd singularity + wget https://github.com/sylabs/singularity/releases/download/v3.9.4/singularity-ce-3.9.4.tar.gz && \ + tar -xzvf singularity-ce-3.9.4.tar.gz && \ + cd singularity-ce-3.9.4 && \ ./mconfig -p /usr/local && \ make -C builddir && \ sudo make -C builddir install diff --git a/CHANGELOG.md b/CHANGELOG.md index c3dba4524..346ed44bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are: The versions coincide with releases on pip. Only major versions will be released as tags on Github. ## [0.0.x](https://github.scom/singularityhub/singularity-hpc/tree/master) (0.0.x) + - Add support for oras pull for singularity (0.0.39) - Bug with setting a nested value (0.0.38) - Adding quotes around tcl descriptions (0.0.37) - fixing bug with container install (does not honor module directory) (0.0.36) diff --git a/docs/getting_started/developer-guide.rst b/docs/getting_started/developer-guide.rst index 1e70befb5..0dec48bb6 100644 --- a/docs/getting_started/developer-guide.rst +++ b/docs/getting_started/developer-guide.rst @@ -239,6 +239,37 @@ to see your environment variables: vsoch +Oras +---- + +As of version 0.0.39 Singularity Registry HPC has support for oras, meaning +we can use the Singularity client to pull an oras endpoint. Instead of using +``docker:`` in the recipe, the container.yaml might look like this: + +.. code-block:: yaml + + oras: ghcr.io/singularityhub/github-ci + url: https://github.com/singularityhub/github-ci/pkgs/container/github-ci + maintainer: '@vsoch' + description: An example SIF on GitHub packages to pull with oras + latest: + latest: sha256:227a917e9ce3a6e1a3727522361865ca92f3147fd202fa1b2e6a7a8220d510b7 + tags: + latest: sha256:227a917e9ce3a6e1a3727522361865ca92f3147fd202fa1b2e6a7a8220d510b7 + + +And then given the ``container.yaml`` file located in ``registry/ghcr.io/singularityhub/github-ci/`` +you would install with shpc and the Singularity container backend as follows: + +.. code-block:: console + + $ shpc install ghcr.io/singularityhub/github-ci + + +**Important**: You should retrieve the image sha from the container registry and +not from the container on your computer, as the two will often be different depending +on metadata added. + Singularity Deploy ------------------ diff --git a/registry/ghcr.io/singularityhub/github-ci/container.yaml b/registry/ghcr.io/singularityhub/github-ci/container.yaml new file mode 100644 index 000000000..733955b44 --- /dev/null +++ b/registry/ghcr.io/singularityhub/github-ci/container.yaml @@ -0,0 +1,8 @@ +oras: ghcr.io/singularityhub/github-ci +url: https://github.com/singularityhub/github-ci/pkgs/container/github-ci +maintainer: '@vsoch' +description: An example SIF on GitHub packages to pull with oras +latest: + latest: sha256:227a917e9ce3a6e1a3727522361865ca92f3147fd202fa1b2e6a7a8220d510b7 +tags: + latest: sha256:227a917e9ce3a6e1a3727522361865ca92f3147fd202fa1b2e6a7a8220d510b7 diff --git a/shpc/main/container/__init__.py b/shpc/main/container/__init__.py index e237f888a..f3c1e1181 100644 --- a/shpc/main/container/__init__.py +++ b/shpc/main/container/__init__.py @@ -90,10 +90,8 @@ def flatname(self): """ Flatten the docker uri into a filesystem appropriate name """ - if self.docker: - return self.docker.replace("/", "-") - elif self.gh: - return self.gh.replace("/", "-") + name = self.docker or self.oras or self.gh + return name.replace("/", "-") @property def name(self): @@ -102,10 +100,8 @@ def name(self): """ from .base import ContainerName - if self.docker: - return ContainerName(self.docker) - elif self.gh: - return ContainerName(self.gh) + name = self.docker or self.oras or self.gh + return ContainerName(name) @property def latest(self): @@ -144,6 +140,19 @@ def get_url(self): def get(self, key, default=None): return self._config.get(key, default) + def get_pull_type(self): + if self.oras: + return "oras" + if self.gh: + return "gh" + return "docker" + + def get_uri(self): + """ + Return the unique resource identifier + """ + return getattr(self, "docker") or getattr(self, "oras") or getattr(self, "gh") + def __getattr__(self, key): """ A direct get of an attribute, but default to None if doesn't exist diff --git a/shpc/main/container/docker.py b/shpc/main/container/docker.py index b92145062..df6685ef6 100644 --- a/shpc/main/container/docker.py +++ b/shpc/main/container/docker.py @@ -56,7 +56,7 @@ def registry_pull(self, module_dir, container_dir, config, tag): """ Pull a container to the library. """ - pull_type = "docker" if getattr(config, "docker") else "gh" + pull_type = config.get_pull_type() if pull_type != "docker": logger.exit("%s only supports Docker (oci registry) pulls." % self.command) diff --git a/shpc/main/container/singularity.py b/shpc/main/container/singularity.py index eab5d0e05..1bd4ac46c 100644 --- a/shpc/main/container/singularity.py +++ b/shpc/main/container/singularity.py @@ -193,7 +193,7 @@ def registry_pull(self, module_dir, container_dir, config, tag): """ Given a module directory, container config, and tag, pull the container """ - pull_type = "docker" if getattr(config, "docker") else "gh" + pull_type = config.get_pull_type() # Preserve name and version of container if it's ever moved container_path = os.path.join( @@ -201,8 +201,12 @@ def registry_pull(self, module_dir, container_dir, config, tag): ) # We pull by the digest - if pull_type == "docker": - container_uri = "docker://%s@%s" % (config.docker, tag.digest) + if pull_type in ["docker", "oras"]: + container_uri = "%s://%s@%s" % ( + pull_type, + config.docker or config.oras, + tag.digest, + ) elif pull_type == "gh": container_uri = "gh://%s/%s:%s" % (config.gh, tag.digest, tag.name) @@ -283,7 +287,7 @@ def pull(self, uri, dest): """ Pull a container to a destination """ - if re.search("^(docker|shub|https)", uri): + if re.search("^(docker|shub|https|oras)", uri): return self._pull_regular(uri, dest) elif uri.startswith("gh://"): return self._pull_github(uri, dest) diff --git a/shpc/main/modules/__init__.py b/shpc/main/modules/__init__.py index ae636b063..9b8fd40c4 100644 --- a/shpc/main/modules/__init__.py +++ b/shpc/main/modules/__init__.py @@ -294,8 +294,8 @@ def install(self, name, tag=None, **kwargs): % (name, "\n".join(config.tags.keys())) ) - # We currently support gh or docker - uri = getattr(config, "docker") or getattr(config, "gh") + # We currently support gh, docker, or oras + uri = config.get_uri() # This is a tag object with name and digest tag = config.tag diff --git a/shpc/main/schemas.py b/shpc/main/schemas.py index 675b4229f..8810c64ec 100644 --- a/shpc/main/schemas.py +++ b/shpc/main/schemas.py @@ -5,7 +5,7 @@ ## ContainerConfig Schema -schema_url = "https://json-schema.org/draft-07/schema/#" +schema_url = "http://json-schema.org/draft-07/schema" # This is also for latest, and a list of tags @@ -69,6 +69,7 @@ containerConfigProperties = { "latest": aliases, "docker": {"type": "string"}, + "oras": {"type": "string"}, "gh": {"type": "string"}, "url": {"type": "string"}, "test": {"type": "string"}, diff --git a/shpc/tests/test_client.py b/shpc/tests/test_client.py index 531ceaac9..f15afb4c8 100644 --- a/shpc/tests/test_client.py +++ b/shpc/tests/test_client.py @@ -91,7 +91,7 @@ def test_features(tmp_path, module_sys, module_file): client.uninstall("python:3.9.2-alpine", force=True) # Now update settings - client.settings.set("container_features:gpu", "nvidia") + client.settings.set("container_features", "gpu:nvidia") # Install known tag, add extra feature of gpu client.install("python:3.9.2-alpine", features=["gpu"]) diff --git a/shpc/tests/test_container.py b/shpc/tests/test_container.py index 4fd5dfdfc..d4cc00537 100644 --- a/shpc/tests/test_container.py +++ b/shpc/tests/test_container.py @@ -31,6 +31,15 @@ def test_pull_gh(tmp_path): assert os.path.exists(result) +def test_pull_oras(tmp_path): + cli = container.SingularityContainer() + + # Test default Singularity pull + image = os.path.join(str(tmp_path), "container.sif") + result = cli.pull("oras://ghcr.io/singularityhub/github-ci:latest", image) + assert os.path.exists(result) + + def test_podman(tmp_path): """Test a singularity container command""" cli = container.PodmanContainer() diff --git a/shpc/version.py b/shpc/version.py index a3137b8f1..a4b7c0b70 100644 --- a/shpc/version.py +++ b/shpc/version.py @@ -2,7 +2,7 @@ __copyright__ = "Copyright 2021-2022, Vanessa Sochat" __license__ = "MPL 2.0" -__version__ = "0.0.38" +__version__ = "0.0.39" AUTHOR = "Vanessa Sochat" NAME = "singularity-hpc" PACKAGE_URL = "https://github.com/singularityhub/singularity-hpc" @@ -16,7 +16,8 @@ # Since we assume wanting Singularity and lmod, we require spython and Jinja2 INSTALL_REQUIRES = ( - ("spython", {"min_version": "0.1.13"}), + # 0.1.18 added support for oras + ("spython", {"min_version": "0.1.18"}), ("Jinja2", {"min_version": None}), ("jsonschema", {"min_version": None}), ("ruamel.yaml", {"min_version": None}),