sources/skopeo: support specifying format

Change the local storage format for containers to the `dir` format.
The `dir` format will be used to retain signatures and manifests.

The remove-signatures option is removed since the storage format now
supports them.

The final move (os.rename()) at the end of the fetch_one() method now
creates the checksum directory if it doesn't exist and moves the child
archive into it, adding to any existing archives that might exist in
other formats (from a previous version downloading a `docker-archive`).

Dropped the .tar suffix from the symlink in the skopeo stage since it's
not necessary and the target of the link might be a directory now.

The parent class exists() method checks if there is a *file* in the
sources cache that matches the checksum.  For containers, this used to
be a file called container-image.tar under a directory that matches the
checksum, so for containers it always returned False.  Added an override
for the skopeo source that checks for the new directory archive.

sources/org.osbuild.skopeo

@@ -79,22 +79,15 @@ class SkopeoSource(sources.SourceService):
             archive_dir = os.path.join(tmpdir, "container-archive")
             os.makedirs(archive_dir)
             os.chmod(archive_dir, 0o755)
-            archive_path = os.path.join(archive_dir, "container-image.tar")
 
             source = f"docker://{imagename}@{digest}"
 
-            # We use the docker format, not oci, because that is the
-            # default return image type of real world registries,
-            # allowing the image to get the same image id as if you
-            # did "podman pull" (rather than converting the image to
-            # oci format, changing the id)
-            destination = f"docker-archive:{archive_path}"
+            # We use the dir format because it is the most powerful in terms of feature support and is the closest to a
+            # direct serialisation of the registry data.
+            dir_name = "container-dir"
+            destination = f"dir:{archive_dir}/{dir_name}"
 
             extra_args = []
-
-            # The archive format can't store signatures, but we still verify them during download
-            extra_args.append("--remove-signatures")
-
             if not tls_verify:
                 extra_args.append("--src-tls-verify=false")
 
@@ -111,9 +104,13 @@ class SkopeoSource(sources.SourceService):
                 raise RuntimeError(
                     f"Downloaded image {imagename}@{digest} has a id of {downloaded_id}, but expected {image_id}")
 
-            # Atomically move download dir into place on successful download
+            # Atomically move download archive into place on successful download
             with ctx.suppress_oserror(errno.ENOTEMPTY, errno.EEXIST):
-                os.rename(archive_dir, f"{self.cache}/{image_id}")
+                os.makedirs(f"{self.cache}/{image_id}", exist_ok=True)
+                os.rename(f"{archive_dir}/{dir_name}", f"{self.cache}/{image_id}/{dir_name}")
+
+    def exists(self, checksum, _desc):
+        return os.path.exists(f"{self.cache}/{checksum}/container-dir")
 
 
 def main():

stages/org.osbuild.skopeo

@@ -82,7 +82,7 @@ def main(inputs, output, options):
         # treats them special, like e.g. /some/path:tag, so we make a symlink to the real name
         # and pass the symlink name to skopeo to make it work with anything
         with tempfile.TemporaryDirectory() as tmpdir:
-            linkname = os.path.join(tmpdir, "image.tar")
+            linkname = os.path.join(tmpdir, "image")
             os.symlink(source, linkname)
 
             if container_format == "dir":