Add support for singularity + oras for a recipe

.circleci/config.yml

@@ -31,9 +31,9 @@ update_go: &update_go
   name: Update Go to 1.14.9
   working_directory: /tmp
   command: |-
-      wget https://dl.google.com/go/go1.14.9.linux-amd64.tar.gz
+      wget https://dl.google.com/go/go1.16.5.linux-amd64.tar.gz
       sudo rm -rf $GOROOT
-      sudo tar -C /usr/local -xzf go1.14.9.linux-amd64.tar.gz
+      sudo tar -C /usr/local -xzf go1.16.5.linux-amd64.tar.gz
       sudo ln -s $GOROOT/bin/go /usr/local/bin/go
 
 fetch_deb_deps: &fetch_deb_deps
@@ -50,9 +50,9 @@ build_singularity: &install_singularity
   working_directory: ~/go/singularity
   command: |-
       cd ~/go
-      wget https://github.com/hpcng/singularity/releases/download/v3.7.1/singularity-3.7.1.tar.gz && \
-      tar -xzvf singularity-3.7.1.tar.gz && \
-      cd singularity
+      wget https://github.com/sylabs/singularity/releases/download/v3.9.4/singularity-ce-3.9.4.tar.gz && \
+      tar -xzvf singularity-ce-3.9.4.tar.gz && \
+      cd singularity-ce-3.9.4 && \
       ./mconfig -p /usr/local && \
       make -C builddir && \
       sudo make -C builddir install

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.scom/singularityhub/singularity-hpc/tree/master) (0.0.x)
+ - Add support for oras pull for singularity (0.0.39)
  - Bug with setting a nested value (0.0.38)
  - Adding quotes around tcl descriptions (0.0.37)
  - fixing bug with container install (does not honor module directory) (0.0.36)

docs/getting_started/developer-guide.rst

@@ -239,6 +239,37 @@ to see your environment variables:
     vsoch
 
 
+Oras
+----
+
+As of version 0.0.39 Singularity Registry HPC has support for oras, meaning
+we can use the Singularity client to pull an oras endpoint. Instead of using
+``docker:`` in the recipe, the container.yaml might look like this:
+
+.. code-block:: yaml
+
+    oras: ghcr.io/singularityhub/github-ci
+    url: https://github.com/singularityhub/github-ci/pkgs/container/github-ci
+    maintainer: '@vsoch'
+    description: An example SIF on GitHub packages to pull with oras
+    latest:
+      latest: sha256:227a917e9ce3a6e1a3727522361865ca92f3147fd202fa1b2e6a7a8220d510b7
+    tags:
+      latest: sha256:227a917e9ce3a6e1a3727522361865ca92f3147fd202fa1b2e6a7a8220d510b7
+
+
+And then given the ``container.yaml`` file located in ``registry/ghcr.io/singularityhub/github-ci/`` 
+you would install with shpc and the Singularity container backend as follows:
+
+.. code-block:: console
+
+    $ shpc install ghcr.io/singularityhub/github-ci
+
+
+**Important**: You should retrieve the image sha from the container registry and 
+not from the container on your computer, as the two will often be different depending
+on metadata added.
+
 Singularity Deploy
 ------------------
 

registry/ghcr.io/singularityhub/github-ci/container.yaml

@@ -0,0 +1,8 @@
+oras: ghcr.io/singularityhub/github-ci
+url: https://github.com/singularityhub/github-ci/pkgs/container/github-ci
+maintainer: '@vsoch'
+description: An example SIF on GitHub packages to pull with oras
+latest:
+  latest: sha256:227a917e9ce3a6e1a3727522361865ca92f3147fd202fa1b2e6a7a8220d510b7
+tags:
+  latest: sha256:227a917e9ce3a6e1a3727522361865ca92f3147fd202fa1b2e6a7a8220d510b7

shpc/main/container/__init__.py

@@ -90,10 +90,8 @@ def flatname(self):
         """
         Flatten the docker uri into a filesystem appropriate name
         """
-        if self.docker:
-            return self.docker.replace("/", "-")
-        elif self.gh:
-            return self.gh.replace("/", "-")
+        name = self.docker or self.oras or self.gh
+        return name.replace("/", "-")
 
     @property
     def name(self):
@@ -102,10 +100,8 @@ def name(self):
         """
         from .base import ContainerName
 
-        if self.docker:
-            return ContainerName(self.docker)
-        elif self.gh:
-            return ContainerName(self.gh)
+        name = self.docker or self.oras or self.gh
+        return ContainerName(name)
 
     @property
     def latest(self):
@@ -144,6 +140,19 @@ def get_url(self):
     def get(self, key, default=None):
         return self._config.get(key, default)
 
+    def get_pull_type(self):
+        if self.oras:
+            return "oras"
+        if self.gh:
+            return "gh"
+        return "docker"
+
+    def get_uri(self):
+        """
+        Return the unique resource identifier
+        """
+        return getattr(self, "docker") or getattr(self, "oras") or getattr(self, "gh")
+
     def __getattr__(self, key):
         """
         A direct get of an attribute, but default to None if doesn't exist

shpc/main/container/docker.py

@@ -56,7 +56,7 @@ def registry_pull(self, module_dir, container_dir, config, tag):
         """
         Pull a container to the library.
         """
-        pull_type = "docker" if getattr(config, "docker") else "gh"
+        pull_type = config.get_pull_type()
         if pull_type != "docker":
             logger.exit("%s only supports Docker (oci registry) pulls." % self.command)
 

shpc/main/container/singularity.py

@@ -193,16 +193,20 @@ def registry_pull(self, module_dir, container_dir, config, tag):
         """
         Given a module directory, container config, and tag, pull the container
         """
-        pull_type = "docker" if getattr(config, "docker") else "gh"
+        pull_type = config.get_pull_type()
 
         # Preserve name and version of container if it's ever moved
         container_path = os.path.join(
             container_dir, "%s-%s-%s.sif" % (config.flatname, tag.name, tag.digest)
         )
 
         # We pull by the digest
-        if pull_type == "docker":
-            container_uri = "docker://%s@%s" % (config.docker, tag.digest)
+        if pull_type in ["docker", "oras"]:
+            container_uri = "%s://%s@%s" % (
+                pull_type,
+                config.docker or config.oras,
+                tag.digest,
+            )
         elif pull_type == "gh":
             container_uri = "gh://%s/%s:%s" % (config.gh, tag.digest, tag.name)
 
@@ -283,7 +287,7 @@ def pull(self, uri, dest):
         """
         Pull a container to a destination
         """
-        if re.search("^(docker|shub|https)", uri):
+        if re.search("^(docker|shub|https|oras)", uri):
             return self._pull_regular(uri, dest)
         elif uri.startswith("gh://"):
             return self._pull_github(uri, dest)

shpc/main/modules/__init__.py

@@ -294,8 +294,8 @@ def install(self, name, tag=None, **kwargs):
                 % (name, "\n".join(config.tags.keys()))
             )
 
-        # We currently support gh or docker
-        uri = getattr(config, "docker") or getattr(config, "gh")
+        # We currently support gh, docker, or oras
+        uri = config.get_uri()
 
         # This is a tag object with name and digest
         tag = config.tag

shpc/main/schemas.py

@@ -5,7 +5,7 @@
 
 ## ContainerConfig Schema
 
-schema_url = "https://json-schema.org/draft-07/schema/#"
+schema_url = "http://json-schema.org/draft-07/schema"
 
 # This is also for latest, and a list of tags
 
@@ -69,6 +69,7 @@
 containerConfigProperties = {
     "latest": aliases,
     "docker": {"type": "string"},
+    "oras": {"type": "string"},
     "gh": {"type": "string"},
     "url": {"type": "string"},
     "test": {"type": "string"},

shpc/tests/test_client.py

@@ -91,7 +91,7 @@ def test_features(tmp_path, module_sys, module_file):
     client.uninstall("python:3.9.2-alpine", force=True)
 
     # Now update settings
-    client.settings.set("container_features:gpu", "nvidia")
+    client.settings.set("container_features", "gpu:nvidia")
 
     # Install known tag, add extra feature of gpu
     client.install("python:3.9.2-alpine", features=["gpu"])

shpc/tests/test_container.py

@@ -31,6 +31,15 @@ def test_pull_gh(tmp_path):
     assert os.path.exists(result)
 
 
+def test_pull_oras(tmp_path):
+    cli = container.SingularityContainer()
+
+    # Test default Singularity pull
+    image = os.path.join(str(tmp_path), "container.sif")
+    result = cli.pull("oras://ghcr.io/singularityhub/github-ci:latest", image)
+    assert os.path.exists(result)
+
+
 def test_podman(tmp_path):
     """Test a singularity container command"""
     cli = container.PodmanContainer()

shpc/version.py

@@ -2,7 +2,7 @@
 __copyright__ = "Copyright 2021-2022, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
-__version__ = "0.0.38"
+__version__ = "0.0.39"
 AUTHOR = "Vanessa Sochat"
 NAME = "singularity-hpc"
 PACKAGE_URL = "https://github.com/singularityhub/singularity-hpc"
@@ -16,7 +16,8 @@
 # Since we assume wanting Singularity and lmod, we require spython and Jinja2
 
 INSTALL_REQUIRES = (
-    ("spython", {"min_version": "0.1.13"}),
+    # 0.1.18 added support for oras
+    ("spython", {"min_version": "0.1.18"}),
     ("Jinja2", {"min_version": None}),
     ("jsonschema", {"min_version": None}),
     ("ruamel.yaml", {"min_version": None}),