Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Adds disable_host_mount suboption to container-engine #1672

Merged
merged 4 commits into from
Jan 20, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions bin/generate_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,9 @@
oneOf:
- enum: [docker, podman]
- type: string
pattern: '^docker; ?create_args:'
pattern: '^docker; ?(create_args|disable_host_mount):'
- type: string
pattern: '^podman; ?create_args:'
pattern: '^podman; ?(create_args|disable_host_mount):'
- type: object
additionalProperties: false
required: [name]
Expand All @@ -74,6 +74,8 @@
type: array
items:
type: string
disable-host-mount:
type: boolean
dependency-versions:
default: pinned
description: Specify how cibuildwheel controls the versions of the tools it uses
Expand Down
26 changes: 21 additions & 5 deletions cibuildwheel/oci_container.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,27 +32,43 @@
class OCIContainerEngineConfig:
name: ContainerEngineName
create_args: Sequence[str] = ()
disable_host_mount: bool = False

@staticmethod
def from_config_string(config_string: str) -> OCIContainerEngineConfig:
config_dict = parse_key_value_string(
config_string, ["name"], ["create_args", "create-args"]
config_string,
["name"],
["create_args", "create-args", "disable_host_mount", "disable-host-mount"],
)
name = " ".join(config_dict["name"])
if name not in {"docker", "podman"}:
msg = f"unknown container engine {name}"
raise ValueError(msg)

name = typing.cast(ContainerEngineName, name)
# some flexibility in the option name to cope with TOML conventions
# some flexibility in the option names to cope with TOML conventions
create_args = config_dict.get("create_args") or config_dict.get("create-args") or []
return OCIContainerEngineConfig(name=name, create_args=create_args)
disable_host_mount_options = (
config_dict.get("disable_host_mount") or config_dict.get("disable-host-mount") or []
)
disable_host_mount = (
strtobool(disable_host_mount_options[-1]) if disable_host_mount_options else False
)

return OCIContainerEngineConfig(
name=name, create_args=create_args, disable_host_mount=disable_host_mount
)

def options_summary(self) -> str | dict[str, str]:
if not self.create_args:
return self.name
else:
return {"name": self.name, "create_args": repr(self.create_args)}
return {
"name": self.name,
"create_args": repr(self.create_args),
"disable_host_mount": str(self.disable_host_mount),
}


DEFAULT_ENGINE = OCIContainerEngineConfig("docker")
Expand Down Expand Up @@ -136,7 +152,7 @@ def __enter__(self) -> OCIContainer:
"--env=SOURCE_DATE_EPOCH",
f"--name={self.name}",
"--interactive",
"--volume=/:/host", # ignored on CircleCI
*(["--volume=/:/host"] if not self.engine.disable_host_mount else []),
*network_args,
*self.engine.create_args,
self.image,
Expand Down
3 changes: 3 additions & 0 deletions cibuildwheel/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,9 @@ def _inner_fmt(k: str, v: Any, table: TableFmt) -> Iterator[str]:
for inner_v in v:
qv = quote_function(inner_v)
yield table["item"].format(k=k, v=qv)
elif isinstance(v, bool):
qv = quote_function(str(v))
yield table["item"].format(k=k, v=qv)
else:
qv = quote_function(v)
yield table["item"].format(k=k, v=qv)
Expand Down
7 changes: 5 additions & 2 deletions cibuildwheel/resources/cibuildwheel.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -172,11 +172,11 @@
},
{
"type": "string",
"pattern": "^docker; ?create_args:"
"pattern": "^docker; ?(create_args|disable_host_mount):"
},
{
"type": "string",
"pattern": "^podman; ?create_args:"
"pattern": "^podman; ?(create_args|disable_host_mount):"
},
{
"type": "object",
Expand All @@ -196,6 +196,9 @@
"items": {
"type": "string"
}
},
"disable-host-mount": {
"type": "boolean"
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions cibuildwheel/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -741,7 +741,7 @@ def parse_key_value_string(
# split by semicolon
fields = [list(group) for k, group in itertools.groupby(parts, lambda x: x == ";") if not k]

result: dict[str, list[str]] = defaultdict(list)
result: defaultdict[str, list[str]] = defaultdict(list)
for field_i, field in enumerate(fields):
# check to see if the option name is specified
field_name, sep, first_value = field[0].partition(":")
Expand All @@ -762,4 +762,4 @@ def parse_key_value_string(

result[field_name] += values

return result
return dict(result)
22 changes: 15 additions & 7 deletions docs/options.md
Original file line number Diff line number Diff line change
Expand Up @@ -1069,8 +1069,8 @@ Auditwheel detects the version of the manylinux / musllinux standard in the imag

Options:

- `docker[;create_args: ...]`
- `podman[;create_args: ...]`
- `docker[;create_args: ...][;disable_host_mount: true/false]`
- `podman[;create_args: ...][;disable_host_mount: true/false]`

Default: `docker`

Expand All @@ -1079,11 +1079,13 @@ Set the container engine to use. Docker is the default, or you can switch to
running and `docker` available on PATH. To use Podman, it needs to be
installed and `podman` available on PATH.

Arguments can be supplied to the container engine. Currently, the only option
that's customisable is 'create_args'. Parameters to create_args are
space-separated strings, which are passed to the container engine on the
command line when it's creating the container. If you want to include spaces
inside a parameter, use shell-style quoting.
Options can be supplied after the name.

| Option name | Description
|---|---
| `create_args` | Space-separated strings, which are passed to the container engine on the command line when it's creating the container. If you want to include spaces inside a parameter, use shell-style quoting.
| `disable_host_mount` | By default, cibuildwheel will mount the root of the host filesystem as a volume at `/host` in the container. If this is undesirable, pass `true` to this option.


!!! tip

Expand All @@ -1104,6 +1106,9 @@ inside a parameter, use shell-style quoting.

# pass command line options to 'docker create'
CIBW_CONTAINER_ENGINE: "docker; create_args: --gpus all"

# disable the /host mount
CIBW_CONTAINER_ENGINE: "docker; disable_host_mount: true"
```

!!! tab examples "pyproject.toml"
Expand All @@ -1115,6 +1120,9 @@ inside a parameter, use shell-style quoting.

# pass command line options to 'docker create'
container-engine = { name = "docker", create-args = ["--gpus", "all"]}

# disable the /host mount
container-engine = { name = "docker", disable-host-mount = true }
```


Expand Down
28 changes: 27 additions & 1 deletion unit_test/oci_container_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from cibuildwheel.environment import EnvironmentAssignmentBash
from cibuildwheel.oci_container import OCIContainer, OCIContainerEngineConfig
from cibuildwheel.util import detect_ci_provider
from cibuildwheel.util import CIProvider, detect_ci_provider

# Test utilities

Expand Down Expand Up @@ -415,3 +415,29 @@ def test_enforce_32_bit(container_engine, image, shell_args):
text=True,
).stdout
assert json.loads(container_args) == shell_args


@pytest.mark.parametrize(
("config", "should_have_host_mount"),
[
("{name}", True),
("{name}; disable_host_mount: false", True),
("{name}; disable_host_mount: true", False),
],
)
def test_disable_host_mount(tmp_path: Path, container_engine, config, should_have_host_mount):
if detect_ci_provider() is CIProvider.circle_ci:
pytest.skip("Skipping test on CircleCI because docker there does not support --volume")

engine = OCIContainerEngineConfig.from_config_string(config.format(name=container_engine.name))

sentinel_file = tmp_path / "sentinel"
sentinel_file.write_text("12345")

with OCIContainer(engine=engine, image=DEFAULT_IMAGE) as container:
host_mount_path = "/host" + str(sentinel_file)
if should_have_host_mount:
assert container.call(["cat", host_mount_path], capture_output=True) == "12345"
else:
with pytest.raises(subprocess.CalledProcessError):
container.call(["cat", host_mount_path], capture_output=True)
25 changes: 23 additions & 2 deletions unit_test/options_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,41 +201,61 @@ def test_toml_environment_quoting(tmp_path: Path, toml_assignment, result_value)


@pytest.mark.parametrize(
("toml_assignment", "result_name", "result_create_args"),
("toml_assignment", "result_name", "result_create_args", "result_disable_host_mount"),
[
(
'container-engine = "podman"',
"podman",
[],
False,
),
(
'container-engine = {name = "podman"}',
"podman",
[],
False,
),
(
'container-engine = "docker; create_args: --some-option"',
"docker",
["--some-option"],
False,
),
(
'container-engine = {name = "docker", create-args = ["--some-option"]}',
"docker",
["--some-option"],
False,
),
(
'container-engine = {name = "docker", create-args = ["--some-option", "value that contains spaces"]}',
"docker",
["--some-option", "value that contains spaces"],
False,
),
(
'container-engine = {name = "docker", create-args = ["--some-option", "value;that;contains;semicolons"]}',
"docker",
["--some-option", "value;that;contains;semicolons"],
False,
),
(
'container-engine = {name = "docker", disable-host-mount = true}',
"docker",
[],
True,
),
(
'container-engine = {name = "docker", disable_host_mount = true}',
"docker",
[],
True,
),
],
)
def test_container_engine_option(tmp_path: Path, toml_assignment, result_name, result_create_args):
def test_container_engine_option(
tmp_path: Path, toml_assignment, result_name, result_create_args, result_disable_host_mount
):
args = CommandLineArguments.defaults()
args.package_dir = tmp_path

Expand All @@ -253,6 +273,7 @@ def test_container_engine_option(tmp_path: Path, toml_assignment, result_name, r

assert parsed_container_engine.name == result_name
assert parsed_container_engine.create_args == result_create_args
assert parsed_container_engine.disable_host_mount == result_disable_host_mount


def test_environment_pass_references():
Expand Down