support podman even when not installed as "docker" alias (#1769)

* support podman even when not installed as "docker" alias

* fix memory usage gathering with podman

* CI: install podman as needed

* podman: also terminate any lingering containers

Author: mr-c

.github/workflows/ci-tests.yml

@@ -136,10 +136,15 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Set up Singularity
+        if: ${{ matrix.container == 'singularity' }}
         uses: eWaterCycle/setup-singularity@v7
         with:
           singularity-version: ${{ env.singularity_version }}
 
+      - name: Set up Podman
+        if: ${{ matrix.container == 'podman' }}
+        run: sudo rm -f /usr/bin/docker ; sudo apt-get install -y podman
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:

cwltool/builder.py

@@ -17,14 +17,15 @@
 from cwl_utils import expression
 from cwl_utils.file_formats import check_format
 from rdflib import Graph
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.avro.schema import Names, Schema, make_avsc_object
 from schema_salad.exceptions import ValidationException
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import convert_to_dict, json_dumps
 from schema_salad.validate import validate
 from typing_extensions import TYPE_CHECKING, Type  # pylint: disable=unused-import
 
+from ruamel.yaml.comments import CommentedMap
+
 from .errors import WorkflowException
 from .loghandler import _logger
 from .mutation import MutationManager

cwltool/command_line_tool.py

@@ -30,7 +30,6 @@
 )
 
 import shellescape
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.avro.schema import Schema
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import file_uri, uri_file_path
@@ -39,14 +38,16 @@
 from schema_salad.validate import validate_ex
 from typing_extensions import TYPE_CHECKING, Type
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
 from .builder import (
     INPUT_OBJ_VOCAB,
     Builder,
     content_limit_respected_read_bytes,
     substitute,
 )
 from .context import LoadingContext, RuntimeContext, getdefault
-from .docker import DockerCommandLineJob
+from .docker import DockerCommandLineJob, PodmanCommandLineJob
 from .errors import UnsupportedRequirement, WorkflowException
 from .flatten import flatten
 from .job import CommandLineJob, JobBase
@@ -460,6 +461,8 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]:
                         raise UnsupportedRequirement(
                             "Both Docker and MPI have been hinted - don't know what to do"
                         )
+            if runtimeContext.podman:
+                return PodmanCommandLineJob
             return DockerCommandLineJob
         if dockerRequired:
             raise UnsupportedRequirement(

cwltool/context.py

@@ -17,13 +17,14 @@
     Union,
 )
 
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.avro.schema import Names
 from schema_salad.ref_resolver import Loader
 from schema_salad.utils import FetcherCallableType
 from typing_extensions import TYPE_CHECKING
 
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
+from ruamel.yaml.comments import CommentedMap
+
 from .builder import Builder
 from .mpi import MpiConfig
 from .mutation import MutationManager

cwltool/cwlrdf.py

@@ -4,10 +4,11 @@
 
 from rdflib import Graph
 from rdflib.query import ResultRow
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.jsonld_context import makerdf
 from schema_salad.utils import ContextType
 
+from ruamel.yaml.comments import CommentedMap
+
 from .cwlviewer import CWLViewer
 from .process import Process
 

cwltool/docker.py

@@ -90,9 +90,10 @@ def __init__(
     ) -> None:
         """Initialize a command line builder using the Docker software container engine."""
         super().__init__(builder, joborder, make_path_mapper, requirements, hints, name)
+        self.docker_exec = "docker"
 
-    @staticmethod
     def get_image(
+        self,
         docker_requirement: Dict[str, str],
         pull_image: bool,
         force_pull: bool,
@@ -117,7 +118,7 @@ def get_image(
 
         for line in (
             subprocess.check_output(  # nosec
-                ["docker", "images", "--no-trunc", "--all"]
+                [self.docker_exec, "images", "--no-trunc", "--all"]
             )
             .decode("utf-8")
             .splitlines()
@@ -151,7 +152,7 @@ def get_image(
         if (force_pull or not found) and pull_image:
             cmd = []  # type: List[str]
             if "dockerPull" in docker_requirement:
-                cmd = ["docker", "pull", str(docker_requirement["dockerPull"])]
+                cmd = [self.docker_exec, "pull", str(docker_requirement["dockerPull"])]
                 _logger.info(str(cmd))
                 subprocess.check_call(cmd, stdout=sys.stderr)  # nosec
                 found = True
@@ -160,7 +161,7 @@ def get_image(
                 with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile:
                     dfile.write(docker_requirement["dockerFile"])
                 cmd = [
-                    "docker",
+                    self.docker_exec,
                     "build",
                     "--tag=%s" % str(docker_requirement["dockerImageId"]),
                     dockerfile_dir,
@@ -169,7 +170,7 @@ def get_image(
                 subprocess.check_call(cmd, stdout=sys.stderr)  # nosec
                 found = True
             elif "dockerLoad" in docker_requirement:
-                cmd = ["docker", "load"]
+                cmd = [self.docker_exec, "load"]
                 _logger.info(str(cmd))
                 if os.path.exists(docker_requirement["dockerLoad"]):
                     _logger.info(
@@ -203,7 +204,7 @@ def get_image(
                 found = True
             elif "dockerImport" in docker_requirement:
                 cmd = [
-                    "docker",
+                    self.docker_exec,
                     "import",
                     str(docker_requirement["dockerImport"]),
                     str(docker_requirement["dockerImageId"]),
@@ -225,8 +226,8 @@ def get_from_requirements(
         force_pull: bool,
         tmp_outdir_prefix: str,
     ) -> Optional[str]:
-        if not shutil.which("docker"):
-            raise WorkflowException("docker executable is not available")
+        if not shutil.which(self.docker_exec):
+            raise WorkflowException(f"{self.docker_exec} executable is not available")
 
         if self.get_image(
             cast(Dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix
@@ -341,10 +342,10 @@ def create_runtime(
                     runtime = [user_space_docker_cmd, "--quiet", "run", "--nobanner"]
             else:
                 runtime = [user_space_docker_cmd, "run"]
-        elif runtimeContext.podman:
-            runtime = ["podman", "run", "-i", "--userns=keep-id"]
         else:
-            runtime = ["docker", "run", "-i"]
+            runtime = [self.docker_exec, "run", "-i"]
+        if runtimeContext.podman:
+            runtime.append("--userns=keep-id")
         self.append_volume(
             runtime, os.path.realpath(self.outdir), self.builder.outdir, writable=True
         )
@@ -460,3 +461,20 @@ def create_runtime(
                 )
 
         return runtime, cidfile_path
+
+
+class PodmanCommandLineJob(DockerCommandLineJob):
+    """Runs a CommandLineJob in a software container using the podman engine."""
+
+    def __init__(
+        self,
+        builder: Builder,
+        joborder: CWLObjectType,
+        make_path_mapper: Callable[..., PathMapper],
+        requirements: List[CWLObjectType],
+        hints: List[CWLObjectType],
+        name: str,
+    ) -> None:
+        """Initialize a command line builder using the Podman software container engine."""
+        super().__init__(builder, joborder, make_path_mapper, requirements, hints, name)
+        self.docker_exec = "podman"

cwltool/job.py

@@ -874,6 +874,7 @@ def run(
                 cidfile,
                 runtimeContext.tmpdir_prefix,
                 not bool(runtimeContext.cidfile_dir),
+                "podman" if runtimeContext.podman else "docker",
             )
         elif runtimeContext.user_space_docker_cmd:
             monitor_function = functools.partial(self.process_monitor)
@@ -884,6 +885,7 @@ def docker_monitor(
         cidfile: str,
         tmpdir_prefix: str,
         cleanup_cidfile: bool,
+        docker_exe: str,
         process,  # type: subprocess.Popen[str]
     ) -> None:
         """Record memory usage of the running Docker container."""
@@ -901,7 +903,7 @@ def docker_monitor(
                         os.remove(cidfile)
                     except OSError as exc:
                         _logger.warning(
-                            "Ignored error cleaning up Docker cidfile: %s", exc
+                            "Ignored error cleaning up %s cidfile: %s", docker_exe, exc
                         )
                     return
             try:
@@ -915,15 +917,19 @@ def docker_monitor(
         stats_file_name = stats_file.name
         try:
             with open(stats_file_name, mode="w") as stats_file_handle:
+                cmds = [docker_exe, "stats"]
+                if "podman" not in docker_exe:
+                    cmds.append("--no-trunc")
+                cmds.extend(["--format", "{{.MemPerc}}", cid])
                 stats_proc = subprocess.Popen(  # nosec
-                    ["docker", "stats", "--no-trunc", "--format", "{{.MemPerc}}", cid],
+                    cmds,
                     stdout=stats_file_handle,
                     stderr=subprocess.DEVNULL,
                 )
                 process.wait()
                 stats_proc.kill()
         except OSError as exc:
-            _logger.warning("Ignored error with docker stats: %s", exc)
+            _logger.warning("Ignored error with %s stats: %s", docker_exe, exc)
             return
         max_mem_percent = 0  # type: float
         mem_percent = 0  # type: float
@@ -938,8 +944,10 @@ def docker_monitor(
                     )
                     if mem_percent > max_mem_percent:
                         max_mem_percent = mem_percent
-                except ValueError:
-                    break
+                except ValueError as exc:
+                    _logger.debug(
+                        "%s stats parsing error in line %s: %s", docker_exe, line, exc
+                    )
         _logger.info(
             "[job %s] Max memory used: %iMiB",
             self.name,

cwltool/load_tool.py

@@ -21,7 +21,6 @@
 )
 
 from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import Loader, file_uri
 from schema_salad.schema import validate_doc
@@ -34,6 +33,8 @@
     json_dumps,
 )
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
 from . import CWL_CONTENT_TYPES, process, update
 from .context import LoadingContext
 from .errors import GraphTargetMissingException

cwltool/main.py

@@ -36,14 +36,15 @@
 import argcomplete
 import coloredlogs
 import pkg_resources  # part of setuptools
-import ruamel.yaml
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
-from ruamel.yaml.main import YAML
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
 from schema_salad.sourceline import cmap, strip_dup_lineno
 from schema_salad.utils import ContextType, FetcherCallableType, json_dumps, yaml_no_ts
 
+import ruamel.yaml
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+from ruamel.yaml.main import YAML
+
 from . import CWL_CONTENT_TYPES, workflow
 from .argparser import arg_parser, generate_parser, get_default_args
 from .context import LoadingContext, RuntimeContext, getdefault
@@ -105,6 +106,8 @@
 )
 from .workflow import Workflow
 
+docker_exe: str
+
 
 def _terminate_processes() -> None:
     """Kill all spawned processes.
@@ -117,6 +120,7 @@ def _terminate_processes() -> None:
     continuing to execute while it kills the processes that they've
     spawned. This may occasionally lead to unexpected behaviour.
     """
+    global docker_exe
     # It's possible that another thread will spawn a new task while
     # we're executing, so it's not safe to use a for loop here.
     while processes_to_kill:
@@ -130,7 +134,7 @@ def _terminate_processes() -> None:
             try:
                 with open(cidfile[0]) as inp_stream:
                     p = subprocess.Popen(  # nosec
-                        ["docker", "kill", inp_stream.read()], shell=False  # nosec
+                        [docker_exe, "kill", inp_stream.read()], shell=False  # nosec
                     )
                     try:
                         p.wait(timeout=10)
@@ -1009,6 +1013,7 @@ def main(
         stderr_handler = _logger.handlers[-1]
     workflowobj = None
     prov_log_handler: Optional[logging.StreamHandler[ProvOut]] = None
+    global docker_exe
     try:
         if args is None:
             if argsl is None:
@@ -1030,6 +1035,10 @@ def main(
         else:
             runtimeContext = runtimeContext.copy()
 
+        if runtimeContext.podman:
+            docker_exe = "podman"
+        else:
+            docker_exe = "docker"
         # If caller parsed its own arguments, it may not include every
         # cwltool option, so fill in defaults to avoid crashing when
         # dereferencing them in args.

cwltool/pack.py

@@ -14,10 +14,11 @@
     cast,
 )
 
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.ref_resolver import Loader, SubLoader
 from schema_salad.utils import ResolveType
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
 from .context import LoadingContext
 from .load_tool import fetch_document, resolve_and_validate_document
 from .process import shortname, uniquename

cwltool/process.py

@@ -35,7 +35,6 @@
 from mypy_extensions import mypyc_attr
 from pkg_resources import resource_stream
 from rdflib import Graph
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.avro.schema import (
     Names,
     Schema,
@@ -50,6 +49,8 @@
 from schema_salad.validate import avro_type_name, validate_ex
 from typing_extensions import TYPE_CHECKING
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
 from .builder import INPUT_OBJ_VOCAB, Builder
 from .context import LoadingContext, RuntimeContext, getdefault
 from .errors import UnsupportedRequirement, WorkflowException

cwltool/procgenerator.py

@@ -1,10 +1,11 @@
 import copy
 from typing import Dict, Optional, Tuple, cast
 
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.exceptions import ValidationException
 from schema_salad.sourceline import indent
 
+from ruamel.yaml.comments import CommentedMap
+
 from .context import LoadingContext, RuntimeContext
 from .errors import WorkflowException
 from .load_tool import load_tool

cwltool/update.py

@@ -11,11 +11,12 @@
     cast,
 )
 
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import Loader
 from schema_salad.sourceline import SourceLine
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
 from .loghandler import _logger
 from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field