From efb6a606d96bfa1a04cb5f25bddd786b3a1374e6 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sun, 7 Jul 2024 15:33:52 +0200 Subject: [PATCH 01/12] Rename finalize_default() to finalize_value() Fits what the function does better. --- mkosi/config.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mkosi/config.py b/mkosi/config.py index 6f7af68af..9716977a0 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -3311,7 +3311,7 @@ def expand_specifiers(text: str, path: Path) -> str: if c == "%": result += "%" elif setting := SETTINGS_LOOKUP_BY_SPECIFIER.get(c): - if (v := finalize_default(setting)) is None: + if (v := finalize_value(setting)) is None: logging.warning( f"Setting {setting.name} specified by specifier '%{c}' in {text} is not yet set, ignoring" ) @@ -3322,7 +3322,7 @@ def expand_specifiers(text: str, path: Path) -> str: for d in specifier.depends: setting = SETTINGS_LOOKUP_BY_DEST[d] - if finalize_default(setting) is None: + if finalize_value(setting) is None: logging.warning( f"Setting {setting.name} which specifier '%{c}' in {text} depends on is not yet set, " "ignoring" @@ -3409,12 +3409,12 @@ def __call__( assert isinstance(v, str) setattr(namespace, s.dest, s.parse(v, getattr(namespace, self.dest, None))) - def finalize_default(setting: ConfigSetting) -> Optional[Any]: + def finalize_value(setting: ConfigSetting) -> Optional[Any]: if (v := getattr(namespace, setting.dest, None)) is not None: return v for d in setting.default_factory_depends: - finalize_default(SETTINGS_LOOKUP_BY_DEST[d]) + finalize_value(SETTINGS_LOOKUP_BY_DEST[d]) # If the setting was assigned the empty string, we don't use any configured default value. if not hasattr(namespace, setting.dest) and setting.dest in defaults: @@ -3475,7 +3475,7 @@ def match_config(path: Path) -> bool: # If we encounter a setting that has not been explicitly configured yet, we assign the default value # first so that we can match on default values for settings. - if finalize_default(s) is None: + if finalize_value(s) is None: result = False else: result = s.match(v, getattr(namespace, s.dest)) @@ -3559,7 +3559,7 @@ def parse_config_one(path: Path, profiles: bool = False) -> bool: setattr(ns, s.dest, s.parse(v, getattr(ns, s.dest, None))) if profiles: - finalize_default(SETTINGS_LOOKUP_BY_DEST["profile"]) + finalize_value(SETTINGS_LOOKUP_BY_DEST["profile"]) profile = getattr(namespace, "profile") immutable_settings.add("Profile") @@ -3584,9 +3584,9 @@ def parse_config_one(path: Path, profiles: bool = False) -> bool: return True - def finalize_defaults() -> None: + def finalize_values() -> None: for s in SETTINGS: - finalize_default(s) + finalize_value(s) images = [] argv = list(argv) @@ -3630,7 +3630,7 @@ def finalize_defaults() -> None: if args.directory is not None: parse_config_one(Path("."), profiles=True) - finalize_default(SETTINGS_LOOKUP_BY_DEST["images"]) + finalize_value(SETTINGS_LOOKUP_BY_DEST["images"]) include = getattr(namespace, "images") immutable_settings.add("Images") @@ -3660,7 +3660,7 @@ def finalize_defaults() -> None: if not parse_config_one(p if p.is_file() else Path(".")): continue - finalize_defaults() + finalize_values() images += [namespace] namespace = ns_copy @@ -3669,7 +3669,7 @@ def finalize_defaults() -> None: if not images: setattr(namespace, "image", None) - finalize_defaults() + finalize_values() images = [namespace] append = True From 19cffa37df2658b9364ac14f7f691ba1fedfcb23 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sun, 7 Jul 2024 15:35:27 +0200 Subject: [PATCH 02/12] Simplify parse_new_includes() We don't need to keep track of the current amount of includes since those includes are already tracked in parsed_includes and will be ignored. Slightly less efficient but this shouldn't matter here. We also store the inode in parsed_includes before we parse the config to make sure we don't try to parse it more than once. --- mkosi/config.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mkosi/config.py b/mkosi/config.py index 9716977a0..68b553c3c 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -3344,13 +3344,11 @@ def expand_specifiers(text: str, path: Path) -> str: @contextlib.contextmanager def parse_new_includes() -> Iterator[None]: - current_num_of_includes = len(getattr(namespace, "include", [])) - try: yield finally: # Parse any includes that were added after yielding. - for p in getattr(namespace, "include", [])[current_num_of_includes:]: + for p in getattr(namespace, "include", []): for c in BUILTIN_CONFIGS: if p == Path(c): path = resources / c @@ -3363,6 +3361,8 @@ def parse_new_includes() -> Iterator[None]: if (st.st_dev, st.st_ino) in parsed_includes: continue + parsed_includes.add((st.st_dev, st.st_ino)) + if any(p == Path(c) for c in BUILTIN_CONFIGS): _, [config] = parse_config(["--directory", "", "--include", os.fspath(path)]) make_executable( @@ -3378,7 +3378,6 @@ def parse_new_includes() -> Iterator[None]: with chdir(path if path.is_dir() else Path.cwd()): parse_config_one(path if path.is_file() else Path(".")) - parsed_includes.add((st.st_dev, st.st_ino)) class ConfigAction(argparse.Action): def __call__( From 8de94b77899006af48d66288c697cc21ce7ace2a Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sun, 7 Jul 2024 15:39:13 +0200 Subject: [PATCH 03/12] Use return value of finalize_value() in one more place Doesn't change behavior, just slightly easier to read. --- mkosi/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkosi/config.py b/mkosi/config.py index 68b553c3c..b0c5cbe7f 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -3474,10 +3474,10 @@ def match_config(path: Path) -> bool: # If we encounter a setting that has not been explicitly configured yet, we assign the default value # first so that we can match on default values for settings. - if finalize_value(s) is None: + if (value := finalize_value(s)) is None: result = False else: - result = s.match(v, getattr(namespace, s.dest)) + result = s.match(v, value) elif m := MATCH_LOOKUP.get(k): result = m.match(v) From 2459740fae46d5b305b6dbf5d244fab5f352e037 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sun, 7 Jul 2024 15:44:16 +0200 Subject: [PATCH 04/12] Drop dead code We don't read configuration for the genkey verb anymore so this can be dropped. --- mkosi/config.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/mkosi/config.py b/mkosi/config.py index b0c5cbe7f..88f8b3786 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -3702,7 +3702,7 @@ def finalize_values() -> None: die("No images defined in mkosi.images/") images = resolve_deps(images, include) - images = [load_config(args, ns) for ns in images] + images = [load_config(ns) for ns in images] return args, tuple(images) @@ -3849,7 +3849,7 @@ def load_args(args: argparse.Namespace) -> Args: return Args.from_namespace(args) -def load_config(args: Args, config: argparse.Namespace) -> Config: +def load_config(config: argparse.Namespace) -> Config: if config.build_dir: config.build_dir = config.build_dir / f"{config.distribution}~{config.release}~{config.architecture}" @@ -3860,17 +3860,6 @@ def load_config(args: Args, config: argparse.Namespace) -> Config: config.kernel_command_line_extra = load_kernel_command_line_extra(config) config.environment = load_environment(config) - if config.secure_boot and args.verb != Verb.genkey: - if config.secure_boot_key is None and config.secure_boot_certificate is None: - die("UEFI SecureBoot enabled, but couldn't find the certificate and private key.", - hint="Consider generating them with 'mkosi genkey'.") - if config.secure_boot_key is None: - die("UEFI SecureBoot enabled, certificate was found, but not the private key.", - hint="Consider placing it in mkosi.key") - if config.secure_boot_certificate is None: - die("UEFI SecureBoot enabled, private key was found, but not the certificate.", - hint="Consider placing it in mkosi.crt") - if config.overlay and not config.base_trees: die("--overlay can only be used with --base-tree") From e8951077890114f2b7613d557d5e86d3f4beee2c Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Mon, 8 Jul 2024 09:28:46 +0200 Subject: [PATCH 05/12] Resolve source path in mount_base_trees() We want to check the extension on the resolved path, not on any symlinks, so let's make sure we resolve the path first. --- mkosi/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 793fb4343..3817a0935 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -102,6 +102,8 @@ def mount_base_trees(context: Context) -> Iterator[None]: for path in context.config.base_trees: d = context.workspace / f"bases/{path.name}-{uuid.uuid4().hex}" + path = path.resolve() + if path.is_dir(): bases += [path] elif can_extract_tar(path): From e1856bdc9e9b7e5b2910cebd1125fabb6155474c Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Mon, 8 Jul 2024 10:15:36 +0200 Subject: [PATCH 06/12] Use subdirectory of build directory for each subimage Instead of sharing the build directory between all images, let's use a subdirectory of the build directory for subimages. This requires us to unshare the user namespace in run_build() before we create the directories so that we always have permissions to create any nested build directories. --- mkosi/__init__.py | 12 ++++++------ mkosi/config.py | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 3817a0935..a9b8f024f 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -4651,6 +4651,12 @@ def run_sync(args: Args, config: Config, *, resources: Path) -> None: def run_build(args: Args, config: Config, *, resources: Path) -> None: + if (uid := os.getuid()) != 0: + become_root() + unshare(CLONE_NEWNS) + if uid == 0: + run(["mount", "--make-rslave", "/"]) + for p in ( config.output_dir, config.cache_dir, @@ -4664,12 +4670,6 @@ def run_build(args: Args, config: Config, *, resources: Path) -> None: p.mkdir(parents=True, exist_ok=True) INVOKING_USER.chown(p) - if (uid := os.getuid()) != 0: - become_root() - unshare(CLONE_NEWNS) - if uid == 0: - run(["mount", "--make-rslave", "/"]) - if config.build_dir: # Make sure the build directory is owned by root (in the user namespace) so that the correct uid-mapping is # applied if it is used in RuntimeTrees= diff --git a/mkosi/config.py b/mkosi/config.py index 88f8b3786..f8abd99d5 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -3851,7 +3851,9 @@ def load_args(args: argparse.Namespace) -> Args: def load_config(config: argparse.Namespace) -> Config: if config.build_dir: - config.build_dir = config.build_dir / f"{config.distribution}~{config.release}~{config.architecture}" + config.build_dir /= config.build_dir / f"{config.distribution}~{config.release}~{config.architecture}" + if config.image: + config.build_dir /= config.image if config.sign: config.checksum = True From 33951627da670df2f38eb6aab3dab196d062871d Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Mon, 8 Jul 2024 12:45:23 +0200 Subject: [PATCH 07/12] Make INVOKING_USER.chown() take /tmp and /var/tmp into account Let's also chown the directory to be user owned if located in /tmp or /var/tmp. --- mkosi/user.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mkosi/user.py b/mkosi/user.py index 823af4fc0..89bfaff3a 100644 --- a/mkosi/user.py +++ b/mkosi/user.py @@ -89,10 +89,11 @@ def rchown(cls, path: Path) -> None: def chown(cls, path: Path) -> None: # If we created a file/directory in a parent directory owned by the invoking user, make sure the path and any # parent directories are owned by the invoking user as well. - if ( - cls.is_regular_user() and - (q := next((parent for parent in path.parents if parent.stat().st_uid == cls.uid), None)) - ): + + def is_valid_dir(path: Path) -> bool: + return path.stat().st_uid == cls.uid or path in (Path("/tmp"), Path("/var/tmp")) + + if cls.is_regular_user() and (q := next((parent for parent in path.parents if is_valid_dir(parent)), None)): os.chown(path, INVOKING_USER.uid, INVOKING_USER.gid) for parent in parents_below(path, q): From 7151b04fb7506756cea2419f7a6fe3a08a161cd5 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Mon, 8 Jul 2024 14:19:42 +0200 Subject: [PATCH 08/12] Revert "action: Remove apparmor disable logic" Turns out this hasn't been shipped in the default image yet. This reverts commit 90cb8d54a0c65f46f0c7462be74a4135576edddf. --- action.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/action.yaml b/action.yaml index 0197b0db9..267e199db 100644 --- a/action.yaml +++ b/action.yaml @@ -38,6 +38,17 @@ runs: sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_unconfined=0 sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_userns=0 + # Both the unix-chkpwd and swtpm profiles are broken (https://gitlab.com/apparmor/apparmor/-/issues/402) so let's + # just disable and remove apparmor completely. It's not relevant in this context anyway. + # TODO: Remove if https://github.com/actions/runner-images/issues/10015 is ever fixed. + - name: Disable and mask apparmor service + shell: bash + run: | + # This command fails with a non-zero error code even though it unloads the apparmor profiles. + # https://gitlab.com/apparmor/apparmor/-/issues/403 + sudo aa-teardown || true + sudo apt-get remove apparmor + - name: Dependencies shell: bash run: | From e0919439b8d1ea229b76cdd4a1a9a9986774c3b8 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Mon, 8 Jul 2024 15:40:25 +0200 Subject: [PATCH 09/12] Use list type for Images= and Dependencies= instead of tuple Also remove the unneeded JSON transformer/parser. --- mkosi/config.py | 8 ++------ tests/test_json.py | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/mkosi/config.py b/mkosi/config.py index f8abd99d5..c8fbe5205 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -1347,8 +1347,8 @@ class Config: profile: Optional[str] include: list[Path] initrd_include: list[Path] - images: tuple[str, ...] - dependencies: tuple[str, ...] + images: list[str] + dependencies: list[str] minimum_version: Optional[GenericVersion] distribution: Distribution @@ -4171,9 +4171,6 @@ def enum_list_transformer(enumlist: list[str], fieldtype: type[list[E]]) -> list enumtype = fieldtype.__args__[0] # type: ignore return [enumtype[e] for e in enumlist] - def str_tuple_transformer(strtup: list[str], fieldtype: list[tuple[str, ...]]) -> tuple[str, ...]: - return tuple(strtup) - def config_drive_transformer(drives: list[dict[str, Any]], fieldtype: type[QemuDrive]) -> list[QemuDrive]: # TODO: exchange for TypeGuard and list comprehension once on 3.10 ret = [] @@ -4217,7 +4214,6 @@ def key_source_transformer(keysource: dict[str, Any], fieldtype: type[KeySource] uuid.UUID: uuid_transformer, Optional[tuple[str, bool]]: root_password_transformer, list[ConfigTree]: config_tree_transformer, - tuple[str, ...]: str_tuple_transformer, Architecture: enum_transformer, BiosBootloader: enum_transformer, ShimBootloader: enum_transformer, diff --git a/tests/test_json.py b/tests/test_json.py index fd44bd3d2..e5427249a 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -382,7 +382,7 @@ def test_config() -> None: compress_output=Compression.bz2, configure_scripts=[Path("/configure")], credentials= {"credkey": "credval"}, - dependencies=("dep1",), + dependencies=["dep1"], distribution=Distribution.fedora, environment={"foo": "foo", "BAR": "BAR", "Qux": "Qux"}, environment_files=[], @@ -396,7 +396,7 @@ def test_config() -> None: image="default", image_id="myimage", image_version="5", - images=("default", "initrd"), + images=["default", "initrd"], include=[], incremental=False, initrd_include=[Path("/foo/bar"),], From a0398f4d5ded428e4dfed2465a513254155b7d64 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Mon, 8 Jul 2024 16:39:32 +0200 Subject: [PATCH 10/12] Don't do caching if we have base trees Instead of doing a complicated scheme with cache overlays which aren't invalidated when the base tree changes, let's not do caching when there are base trees. We assume that if there's a base tree only a minimal amount of extra packages is installed that is sufficiently fast without caching. --- mkosi/__init__.py | 55 +++++++++++++---------------------------------- 1 file changed, 15 insertions(+), 40 deletions(-) diff --git a/mkosi/__init__.py b/mkosi/__init__.py index a9b8f024f..aa625c002 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -330,20 +330,6 @@ def configure_autologin(context: Context) -> None: "--keep-baud 115200,57600,38400,9600 -") -@contextlib.contextmanager -def mount_cache_overlay(context: Context, cached: bool) -> Iterator[None]: - if not context.config.incremental or not context.config.base_trees or context.config.overlay or cached: - yield - return - - d = context.workspace / "cache-overlay" - with umask(~0o755): - d.mkdir(exist_ok=True) - - with mount_overlay([context.root], d, context.root): - yield - - @contextlib.contextmanager def mount_build_overlay(context: Context, volatile: bool = False) -> Iterator[Path]: d = context.workspace / "build-overlay" @@ -3237,7 +3223,7 @@ def need_build_overlay(config: Config) -> bool: def save_cache(context: Context) -> None: - if not context.config.incremental or context.config.overlay: + if not context.config.incremental or context.config.base_trees or context.config.overlay: return final, build, manifest = cache_tree_paths(context.config) @@ -3245,20 +3231,11 @@ def save_cache(context: Context) -> None: with complete_step("Installing cache copies"): rmtree(final, sandbox=context.sandbox) - # We only use the cache-overlay directory for caching if we have a base tree, otherwise we just - # cache the root directory. - if (context.workspace / "cache-overlay").exists(): - move_tree( - context.workspace / "cache-overlay", final, - use_subvolumes=context.config.use_subvolumes, - sandbox=context.sandbox, - ) - else: - move_tree( - context.root, final, - use_subvolumes=context.config.use_subvolumes, - sandbox=context.sandbox, - ) + move_tree( + context.root, final, + use_subvolumes=context.config.use_subvolumes, + sandbox=context.sandbox, + ) if need_build_overlay(context.config) and (context.workspace / "build-overlay").exists(): rmtree(build, sandbox=context.sandbox) @@ -3279,7 +3256,7 @@ def save_cache(context: Context) -> None: def have_cache(config: Config) -> bool: - if not config.incremental or config.overlay: + if not config.incremental or config.base_trees or config.overlay: return False final, build, manifest = cache_tree_paths(config) @@ -3852,21 +3829,19 @@ def build_image(context: Context) -> None: install_base_trees(context) cached = reuse_cache(context) - with mount_cache_overlay(context, cached): - copy_repository_metadata(context) + copy_repository_metadata(context) context.config.distribution.setup(context) install_package_directories(context) if not cached: - with mount_cache_overlay(context, cached): - install_skeleton_trees(context) - install_distribution(context) - run_prepare_scripts(context, build=False) - install_build_packages(context) - run_prepare_scripts(context, build=True) - fixup_vmlinuz_location(context) - run_depmod(context, cache=True) + install_skeleton_trees(context) + install_distribution(context) + run_prepare_scripts(context, build=False) + install_build_packages(context) + run_prepare_scripts(context, build=True) + fixup_vmlinuz_location(context) + run_depmod(context, cache=True) save_cache(context) reuse_cache(context) From 8aefea6923dd049d060eea7e88f9054b674b432e Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Mon, 8 Jul 2024 19:41:16 +0200 Subject: [PATCH 11/12] Fix empty strings for lists and dicts Parse functions should return None to pick the default value. Also, we don't get any values at all if unescape=True and the empty string is passed so make sure we handle that case as well. --- mkosi/config.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/mkosi/config.py b/mkosi/config.py index c8fbe5205..eda345539 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -778,18 +778,20 @@ def config_parse_list(value: Optional[str], old: Optional[list[Any]]) -> Optiona if value is None: return [] + # Empty strings reset the list. + if unescape: lex = shlex.shlex(value, posix=True) lex.whitespace_split = True lex.whitespace = f"\n{delimiter}" lex.commenters = "" values = list(lex) + if reset and not values: + return None else: values = value.replace(delimiter, "\n").split("\n") - - # Empty strings reset the list. - if reset and len(values) == 1 and values[0] == "": - return [] + if reset and len(values) == 1 and values[0] == "": + return None return new + [parse(v) for v in values if v] @@ -855,18 +857,20 @@ def config_parse_dict(value: Optional[str], old: Optional[dict[str, Any]]) -> Op return new + # Empty strings reset the dict. + if unescape: lex = shlex.shlex(value, posix=True) lex.whitespace_split = True lex.whitespace = f"\n{delimiter}" lex.commenters = "" values = list(lex) + if reset and not values: + return None else: values = value.replace(delimiter, "\n").split("\n") - - # Empty strings reset the dict. - if reset and len(values) == 1 and values[0] == "": - return {} + if reset and len(values) == 1 and values[0] == "": + return None return new | dict(parse(v) for v in values if v) From 1b8f7f240dfdc85bc7bdf2b3aea3c590d2203eba Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sun, 7 Jul 2024 17:22:40 +0200 Subject: [PATCH 12/12] Rework configuration parsing (again) As explained in #2846, there are multiple issues with the current implementation of mkosi.images. Let's take what we learned from the default initrd and the default tools tree and apply it mkosi.images. Specifically, all issues arise from the fact that we apply every option from the global configuration (including CLI arguments) to the images from mkosi.images/. To avoid the issues that arise from this (e.g --package abc installing abc in all images), we made configuration values override CLI arguments again so that we could override faulty CLI arguments again in subimages so that they would only apply to the main image (e.g. set Format= explicitly for each subimage so that --format on the command line only applies to the main image). Because we still wanted to allow configurable settings that can be modified via the command line, we introduced the default specifier '@' which can be prefixed to a setting to set a default value instead of overriding the value. The '@' specifier is generally used in the global image independent configuration to specify default values that can be overridden from the command line. This specifier has led to a lot of confusion, along with the behavior that the CLI does not override the configuration. From the default tools tree and default initrd, we learned that what works very well is to only have specific settings from the main image configuration apply to the default tools tree and default initrd. For example, the distribution, release, mirror and architecture should be the same for the main image and the initrd, but the packages from the main image should not all be installed in the initrd. We can apply this idea to the images from mkosi.images/ as well, if we introduce the assumption that all images defined in mkosi.images are subimages intended to be included in some way or form in the main image. This assumption allows us to divide all settings into either image specific settings or "universal" settings that should apply to the main image and all its subimages. The universal settings are passed on to each subimage. The image specific settings are not. This idea also allows us to define the "main" image outside of mkosi.images again. Since only "universal" settings are passed on, we can safely define an output format and such again in the global configuration, as we know this won't be passed on to subimages. It also allows us to make CLI arguments override configuration again. Since there is no need anymore for subimages to override the CLI configuration as inappropriate CLI configuration such as extra packages will only apply to the main image and not any subimages from mkosi.images/. Because CLI configuration overrides file configuration again, we also don't need the '@' specifier anymore, as default values can simply be set without '@', since the CLI will override the configuration file values by default. We also lose the need for --append, because the sole use for --append was again to override file based configuration. Note that configuration from mkosi.local.conf is special in that it should override settings from other configuration files, but not settings that are specified on the CLI. This commit implements all of what's mentioned above, specifically: - CLI configuration now always trumps file based configuration. - The '@' specifier is dropped automatically during parsing - The main image is now always added from global configuration, even if there are images in mkosi.images/. The main image is always built last, and cannot be used as a dependency in the Dependencies= setting for images defined in mkosi.images/. - The Dependencies= setting for the main image now is used to specify which subimages from mkosi.images/ to build. By default all subimages are built. - A universal tag is introduced for settings and appropriate settings are marked as universal. Universal settings are passed on from the main image configuration to subimage configuration. - The Images= setting is removed, as it's role is replaced by Dependencies=. - The old name mkosi.presets and the Preset section name are removed as they have been deprecated for a long time now. - The config parsing tests are extended to cover more cases. - All builtin configuration is adapted to stop using the '@' specifier. - The documentation is updated in accordance with the changes. --- docs/sysext.md | 17 +- mkosi.conf | 12 +- mkosi.conf.d/15-bootable.conf | 2 +- mkosi.conf.d/15-memory.conf | 2 +- mkosi.conf.d/15-x86-64.conf | 2 +- mkosi.conf.d/20-centos.conf | 4 +- mkosi.conf.d/20-debian/mkosi.conf | 2 +- mkosi.conf.d/20-fedora/mkosi.conf | 2 +- mkosi.conf.d/20-opensuse/mkosi.conf | 4 +- mkosi.conf.d/20-rhel-ubi.conf | 2 +- mkosi.conf.d/20-ubuntu/mkosi.conf | 2 +- mkosi/config.py | 415 ++++++++++++++---------- mkosi/resources/mkosi-initrd/mkosi.conf | 8 +- mkosi/resources/mkosi-tools/mkosi.conf | 2 +- mkosi/resources/mkosi.md | 101 ++++-- mkosi/util.py | 4 + tests/test_config.py | 132 ++++++-- tests/test_json.py | 7 - 18 files changed, 447 insertions(+), 273 deletions(-) diff --git a/docs/sysext.md b/docs/sysext.md index d43ba5453..edfe48e95 100644 --- a/docs/sysext.md +++ b/docs/sysext.md @@ -62,7 +62,7 @@ BaseTrees=%O/base Packages=btrfs-progs ``` -`BaseTrees=` point to our base image and `Overlay=yes` instructs mkosi +`BaseTrees=` points to our base image and `Overlay=yes` instructs mkosi to only package the files added on top of the base tree. We can't sign the extension image without a key, so let's generate one @@ -72,20 +72,19 @@ key will need to be loaded into your kernel keyring either at build time or via MOK for systemd to accept the system extension at runtime as trusted. -Finally, you can build the base image and the extensions by running +Finally, you can build the base image and the extension by running `mkosi -f`. You'll find `btrfs.raw` in `mkosi.output` which is the -extension image. +extension image. You'll also find the main image `image.raw` there but +it will be almost empty. -If you want to package up the base image into another format, for -example an initrd, we can do that by adding the following to -`mkosi.images/initrd/mkosi.conf`: +What we can do now is package up the base image as the main image, but +in another format, for example an initrd, we can do that by adding the +following to `mkosi.conf`: ```conf -[Config] -Dependencies=base - [Output] Format=cpio +Output=initrd [Content] MakeInitrd=yes diff --git a/mkosi.conf b/mkosi.conf index 000536b17..9c57debc7 100644 --- a/mkosi.conf +++ b/mkosi.conf @@ -3,14 +3,14 @@ [Output] # These images are (among other things) used for running mkosi which means we need some disk space available so # default to directory output where disk space isn't a problem. -@Format=directory -@CacheDirectory=mkosi.cache -@OutputDirectory=mkosi.output +Format=directory +CacheDirectory=mkosi.cache +OutputDirectory=mkosi.output [Content] Autologin=yes -@SELinuxRelabel=no -@ShimBootloader=unsigned +SELinuxRelabel=no +ShimBootloader=unsigned BuildSources=. BuildSourcesEphemeral=yes @@ -36,4 +36,4 @@ RemoveFiles= KernelCommandLine=enforcing=0 [Host] -@QemuMem=4G +QemuMem=4G diff --git a/mkosi.conf.d/15-bootable.conf b/mkosi.conf.d/15-bootable.conf index 5622c102b..4d5e79777 100644 --- a/mkosi.conf.d/15-bootable.conf +++ b/mkosi.conf.d/15-bootable.conf @@ -9,4 +9,4 @@ Architecture=|x86-64 Architecture=|arm64 [Content] -@Bootable=yes +Bootable=yes diff --git a/mkosi.conf.d/15-memory.conf b/mkosi.conf.d/15-memory.conf index 080022ed1..f260df561 100644 --- a/mkosi.conf.d/15-memory.conf +++ b/mkosi.conf.d/15-memory.conf @@ -6,4 +6,4 @@ Format=|uki Format=|cpio [Host] -@QemuMem=8G +QemuMem=8G diff --git a/mkosi.conf.d/15-x86-64.conf b/mkosi.conf.d/15-x86-64.conf index 1b0c4b30c..c71669238 100644 --- a/mkosi.conf.d/15-x86-64.conf +++ b/mkosi.conf.d/15-x86-64.conf @@ -8,4 +8,4 @@ Architecture=x86-64 ToolsTreeDistribution=!opensuse [Content] -@BiosBootloader=grub +BiosBootloader=grub diff --git a/mkosi.conf.d/20-centos.conf b/mkosi.conf.d/20-centos.conf index eccb74ff8..504b9396f 100644 --- a/mkosi.conf.d/20-centos.conf +++ b/mkosi.conf.d/20-centos.conf @@ -6,10 +6,10 @@ Distribution=|alma Distribution=|rocky [Distribution] -@Release=9 +Release=9 [Content] # CentOS Stream 10 does not ship an unsigned shim -@ShimBootloader=none +ShimBootloader=none Packages= linux-firmware diff --git a/mkosi.conf.d/20-debian/mkosi.conf b/mkosi.conf.d/20-debian/mkosi.conf index 8ead9b513..62a185682 100644 --- a/mkosi.conf.d/20-debian/mkosi.conf +++ b/mkosi.conf.d/20-debian/mkosi.conf @@ -4,7 +4,7 @@ Distribution=debian [Distribution] -@Release=testing +Release=testing Repositories=non-free-firmware [Content] diff --git a/mkosi.conf.d/20-fedora/mkosi.conf b/mkosi.conf.d/20-fedora/mkosi.conf index 1a05c7cb3..977210f0f 100644 --- a/mkosi.conf.d/20-fedora/mkosi.conf +++ b/mkosi.conf.d/20-fedora/mkosi.conf @@ -4,7 +4,7 @@ Distribution=fedora [Distribution] -@Release=rawhide +Release=rawhide [Content] Packages= diff --git a/mkosi.conf.d/20-opensuse/mkosi.conf b/mkosi.conf.d/20-opensuse/mkosi.conf index d7667bdf4..578871308 100644 --- a/mkosi.conf.d/20-opensuse/mkosi.conf +++ b/mkosi.conf.d/20-opensuse/mkosi.conf @@ -4,11 +4,11 @@ Distribution=opensuse [Distribution] -@Release=tumbleweed +Release=tumbleweed [Content] # OpenSUSE does not ship an unsigned shim -@ShimBootloader=none +ShimBootloader=none Packages= bash diffutils diff --git a/mkosi.conf.d/20-rhel-ubi.conf b/mkosi.conf.d/20-rhel-ubi.conf index 088eda43a..cc4940a0c 100644 --- a/mkosi.conf.d/20-rhel-ubi.conf +++ b/mkosi.conf.d/20-rhel-ubi.conf @@ -4,7 +4,7 @@ Distribution=rhel-ubi [Distribution] -@Release=9 +Release=9 [Content] Bootable=no diff --git a/mkosi.conf.d/20-ubuntu/mkosi.conf b/mkosi.conf.d/20-ubuntu/mkosi.conf index 4a112cad2..43edd67c7 100644 --- a/mkosi.conf.d/20-ubuntu/mkosi.conf +++ b/mkosi.conf.d/20-ubuntu/mkosi.conf @@ -4,7 +4,7 @@ Distribution=ubuntu [Distribution] -@Release=noble +Release=noble Repositories=universe [Content] diff --git a/mkosi/config.py b/mkosi/config.py index eda345539..127daba1b 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -101,9 +101,6 @@ def needs_build(self) -> bool: def needs_root(self) -> bool: return self in (Verb.shell, Verb.boot, Verb.burn) - def needs_credentials(self) -> bool: - return self in (Verb.summary, Verb.qemu, Verb.boot, Verb.shell) - def needs_config(self) -> bool: return self not in (Verb.help, Verb.genkey, Verb.documentation, Verb.dependencies) @@ -730,6 +727,24 @@ def config_default_proxy_url(namespace: argparse.Namespace) -> Optional[str]: return None +def config_default_dependencies(namespace: argparse.Namespace) -> Optional[list[str]]: + if namespace.directory is None or not Path("mkosi.images").exists(): + return [] + + if namespace.image: + return [] + + dependencies = [] + + for p in sorted(Path("mkosi.images").iterdir()): + if not p.is_dir() and not p.suffix == ".conf": + continue + + dependencies += [p.name.removesuffix(".conf")] + + return dependencies + + def make_enum_parser(type: type[StrEnum]) -> Callable[[str], StrEnum]: def parse_enum(value: str) -> StrEnum: try: @@ -1146,8 +1161,8 @@ class ConfigSetting: paths: tuple[str, ...] = () path_read_text: bool = False path_secret: bool = False - path_default: bool = True specifier: str = "" + universal: bool = False # settings for argparse short: Optional[str] = None @@ -1278,7 +1293,6 @@ class Args: auto_bump: bool doc_format: DocFormat json: bool - append: bool @classmethod def default(cls) -> "Args": @@ -1351,7 +1365,6 @@ class Config: profile: Optional[str] include: list[Path] initrd_include: list[Path] - images: list[str] dependencies: list[str] minimum_version: Optional[GenericVersion] @@ -1826,20 +1839,14 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple help="Build the specified profile", parse=config_parse_profile, match=config_make_string_matcher(), - ), - ConfigSetting( - dest="images", - compat_names=("Presets",), - long="--image", - section="Config", - parse=config_make_list_parser(delimiter=","), - help="Specify which images to build", + universal=True, ), ConfigSetting( dest="dependencies", long="--dependency", section="Config", parse=config_make_list_parser(delimiter=","), + default_factory=config_default_dependencies, help="Specify other images that this image depends on", ), ConfigSetting( @@ -1855,7 +1862,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Config", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), paths=("mkosi.configure",), - path_default=False, help="Configure script to run before doing anything", ), ConfigSetting( @@ -1866,8 +1872,9 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_make_enum_parser(Distribution), match=config_make_enum_matcher(Distribution), default_factory=config_default_distribution, - choices=Distribution.values(), + choices=Distribution.choices(), help="Distribution to install", + universal=True, ), ConfigSetting( dest="release", @@ -1879,6 +1886,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple default_factory=config_default_release, default_factory_depends=("distribution",), help="Distribution release to install", + universal=True, ), ConfigSetting( dest="architecture", @@ -1887,19 +1895,22 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_make_enum_parser(Architecture), match=config_make_enum_matcher(Architecture), default=Architecture.native(), - choices=Architecture.values(), + choices=Architecture.choices(), help="Override the architecture of installation", + universal=True, ), ConfigSetting( dest="mirror", short="-m", section="Distribution", help="Distribution mirror to use", + universal=True, ), ConfigSetting( dest="local_mirror", section="Distribution", help="Use a single local, flat and plain mirror to build the image", + universal=True, ), ConfigSetting( dest="repository_key_check", @@ -1909,6 +1920,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple default=True, parse=config_parse_boolean, help="Controls signature and key checks on repositories", + universal=True, ), ConfigSetting( dest="repositories", @@ -1916,6 +1928,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Distribution", parse=config_make_list_parser(delimiter=","), help="Repositories to use", + universal=True, ), ConfigSetting( dest="cacheonly", @@ -1925,7 +1938,8 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_make_enum_parser_with_boolean(Cacheonly, yes=Cacheonly.always, no=Cacheonly.auto), default=Cacheonly.auto, help="Only use the package cache when installing packages", - choices=Cacheonly.values(), + choices=Cacheonly.choices(), + universal=True, ), ConfigSetting( dest="package_manager_trees", @@ -1936,6 +1950,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple default_factory=lambda ns: ns.skeleton_trees, default_factory_depends=("skeleton_trees",), help="Use a package manager tree to configure the package manager", + universal=True, ), ConfigSetting( @@ -1948,7 +1963,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_make_enum_parser(OutputFormat), match=config_make_enum_matcher(OutputFormat), default=OutputFormat.disk, - choices=OutputFormat.values(), + choices=OutputFormat.choices(), help="Output Format", ), ConfigSetting( @@ -2000,6 +2015,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_make_path_parser(required=False), paths=("mkosi.output",), help="Output directory", + universal=True, ), ConfigSetting( dest="workspace_dir", @@ -2008,6 +2024,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Output", parse=config_make_path_parser(required=False), help="Workspace directory", + universal=True, ), ConfigSetting( dest="cache_dir", @@ -2017,6 +2034,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_make_path_parser(required=False), paths=("mkosi.cache",), help="Incremental cache directory", + universal=True, ), ConfigSetting( dest="package_cache_dir", @@ -2025,6 +2043,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Output", parse=config_make_path_parser(required=False), help="Package cache directory", + universal=True, ), ConfigSetting( dest="build_dir", @@ -2034,6 +2053,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_make_path_parser(required=False), paths=("mkosi.builddir",), help="Path to use as persistent build directory", + universal=True, ), ConfigSetting( dest="image_version", @@ -2043,6 +2063,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple help="Set version for image", paths=("mkosi.version",), path_read_text=True, + universal=True, ), ConfigSetting( dest="image_id", @@ -2050,6 +2071,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Output", specifier="i", help="Set ID for image", + universal=True, ), ConfigSetting( dest="split_artifacts", @@ -2074,6 +2096,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Output", parse=config_parse_sector_size, help="Set the disk image sector size", + universal=True, ), ConfigSetting( dest="repart_offline", @@ -2081,6 +2104,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_parse_boolean, help="Build disk images without using loopback devices", default=True, + universal=True, ), ConfigSetting( dest="overlay", @@ -2097,6 +2121,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Output", parse=config_parse_feature, help="Use btrfs subvolumes for faster directory operations where possible", + universal=True, ), ConfigSetting( dest="seed", @@ -2113,7 +2138,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Output", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), paths=("mkosi.clean",), - path_default=False, help="Clean script to run after cleanup", ), @@ -2149,8 +2173,8 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Content", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), paths=("mkosi.packages",), - path_default=False, help="Specify a directory containing extra packages", + universal=True, ), ConfigSetting( dest="with_recommends", @@ -2184,7 +2208,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Content", parse=config_make_list_parser(delimiter=",", parse=make_tree_parser()), paths=("mkosi.skeleton", "mkosi.skeleton.tar"), - path_default=False, help="Use a skeleton tree to bootstrap the image before installing anything", ), ConfigSetting( @@ -2194,7 +2217,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Content", parse=config_make_list_parser(delimiter=",", parse=make_tree_parser()), paths=("mkosi.extra", "mkosi.extra.tar"), - path_default=False, help="Copy an extra tree on top of image", ), ConfigSetting( @@ -2227,6 +2249,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple default_factory=config_default_source_date_epoch, default_factory_depends=("environment",), help="Set the $SOURCE_DATE_EPOCH timestamp", + universal=True, ), ConfigSetting( dest="sync_scripts", @@ -2235,7 +2258,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Content", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), paths=("mkosi.sync",), - path_default=False, help="Sync script to run before starting the build", ), ConfigSetting( @@ -2245,7 +2267,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Content", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), paths=("mkosi.prepare", "mkosi.prepare.chroot"), - path_default=False, help="Prepare script to run inside the image before it is cached", compat_names=("PrepareScript",), ), @@ -2256,7 +2277,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Content", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), paths=("mkosi.build", "mkosi.build.chroot"), - path_default=False, help="Build script to run inside image", compat_names=("BuildScript",), ), @@ -2268,7 +2288,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Content", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), paths=("mkosi.postinst", "mkosi.postinst.chroot"), - path_default=False, help="Postinstall script to run inside image", compat_names=("PostInstallationScript",), ), @@ -2279,7 +2298,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Content", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), paths=("mkosi.finalize", "mkosi.finalize.chroot"), - path_default=False, help="Postinstall script to run outside image", compat_names=("FinalizeScript",), ), @@ -2291,7 +2309,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Content", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), paths=("mkosi.postoutput",), - path_default=False, help="Output postprocessing script to run outside image", ), ConfigSetting( @@ -2326,7 +2343,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Content", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), paths=("mkosi.env",), - path_default=False, help="Enviroment files to set when running scripts", ), ConfigSetting( @@ -2361,7 +2377,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple dest="bootloader", section="Content", parse=config_make_enum_parser(Bootloader), - choices=Bootloader.values(), + choices=Bootloader.choices(), default=Bootloader.systemd_boot, help="Specify which UEFI bootloader to use", ), @@ -2369,7 +2385,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple dest="bios_bootloader", section="Content", parse=config_make_enum_parser(BiosBootloader), - choices=BiosBootloader.values(), + choices=BiosBootloader.choices(), default=BiosBootloader.none, help="Specify which BIOS bootloader to use", ), @@ -2377,7 +2393,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple dest="shim_bootloader", section="Content", parse=config_make_enum_parser(ShimBootloader), - choices=ShimBootloader.values(), + choices=ShimBootloader.choices(), default=ShimBootloader.none, help="Specify whether to use shim", ), @@ -2622,7 +2638,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Validation", parse=config_make_enum_parser(SecureBootSignTool), default=SecureBootSignTool.auto, - choices=SecureBootSignTool.values(), + choices=SecureBootSignTool.choices(), help="Tool to use for signing PE binaries for secure boot", ), ConfigSetting( @@ -2632,6 +2648,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_parse_key, paths=("mkosi.key",), help="Private key for signing verity signature", + universal=True, ), ConfigSetting( dest="verity_key_source", @@ -2640,6 +2657,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_parse_key_source, default=KeySource(type=KeySource.Type.file), help="The source to use to retrieve the verity signing key", + universal=True, ), ConfigSetting( dest="verity_certificate", @@ -2648,6 +2666,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_make_path_parser(), paths=("mkosi.crt",), help="Certificate for signing verity signature in X509 format", + universal=True, ), ConfigSetting( dest="sign_expected_pcr", @@ -2693,6 +2712,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple default_factory_depends=("environment",), metavar="URL", help="Set the proxy to use", + universal=True, ), ConfigSetting( dest="proxy_exclude", @@ -2700,6 +2720,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple metavar="HOST", parse=config_make_list_parser(delimiter=","), help="Don't use the configured proxy for the specified host(s)", + universal=True, ), ConfigSetting( dest="proxy_peer_certificate", @@ -2710,12 +2731,14 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple "/etc/ssl/certs/ca-certificates.crt", ), help="Set the proxy peer certificate", + universal=True, ), ConfigSetting( dest="proxy_client_certificate", section="Host", parse=config_make_path_parser(secret=True), help="Set the proxy client certificate", + universal=True, ), ConfigSetting( dest="proxy_client_key", @@ -2724,6 +2747,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple default_factory_depends=("proxy_client_certificate",), parse=config_make_path_parser(secret=True), help="Set the proxy client key", + universal=True, ), ConfigSetting( dest="incremental", @@ -2733,6 +2757,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Host", parse=config_parse_boolean, help="Make use of and generate intermediary cache images", + universal=True, ), ConfigSetting( dest="nspawn_settings", @@ -2751,6 +2776,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Host", parse=config_make_list_parser(delimiter=",", parse=make_path_parser()), help="List of comma-separated paths to look for programs before looking in PATH", + universal=True, ), ConfigSetting( dest="ephemeral", @@ -2769,7 +2795,6 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_make_dict_parser(delimiter=" ", parse=parse_credential, allow_paths=True, unescape=True), help="Pass a systemd credential to systemd-nspawn or qemu", paths=("mkosi.credentials",), - path_default=False, ), ConfigSetting( dest="kernel_command_line_extra", @@ -2785,6 +2810,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple section="Host", parse=config_parse_boolean, help="Set ACLs on generated directories to permit the user running mkosi to remove them", + universal=True, ), ConfigSetting( dest="tools_tree", @@ -2795,13 +2821,14 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple help="Look up programs to execute inside the given tree", nargs="?", const="default", + universal=True, ), ConfigSetting( dest="tools_tree_distribution", section="Host", parse=config_make_enum_parser(Distribution), match=config_make_enum_matcher(Distribution), - choices=Distribution.values(), + choices=Distribution.choices(), default_factory_depends=("distribution",), default_factory=lambda ns: ns.distribution.default_tools_tree_distribution(), help="Set the distribution to use for the default tools tree", @@ -2819,7 +2846,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple dest="tools_tree_mirror", metavar="MIRROR", section="Host", - default_factory_depends=("distribution", "tools_tree_distribution"), + default_factory_depends=("distribution", "mirror", "tools_tree_distribution"), default_factory=lambda ns: ns.mirror if ns.mirror and ns.distribution == ns.tools_tree_distribution else None, help="Set the mirror to use for the default tools tree", ), @@ -2854,6 +2881,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_parse_boolean, help="Use certificates from the tools tree", default=True, + universal=True, ), ConfigSetting( dest="runtime_trees", @@ -2881,7 +2909,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple dest="runtime_network", section="Host", parse=config_make_enum_parser(Network), - choices=Network.values(), + choices=Network.choices(), help="Set networking backend to use when booting the image", default=Network.user, ), @@ -2919,7 +2947,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple dest="vmm", name="VirtualMachineMonitor", section="Host", - choices=Vmm.values(), + choices=Vmm.choices(), parse=config_make_enum_parser(Vmm), default=Vmm.qemu, help="Set the virtual machine monitor to use for mkosi qemu", @@ -3009,7 +3037,7 @@ def parse_ini(path: Path, only_sections: Collection[str] = ()) -> Iterator[tuple parse=config_make_enum_parser(QemuFirmware), default=QemuFirmware.auto, help="Set qemu firmware to use", - choices=QemuFirmware.values(), + choices=QemuFirmware.choices(), ), ConfigSetting( dest="qemu_firmware_variables", @@ -3198,12 +3226,6 @@ def create_argument_parser(action: type[argparse.Action], chdir: bool = True) -> action="store_true", default=False, ) - parser.add_argument( - "--append", - help="All settings passed after this argument will be parsed after all configuration files are parsed", - action="store_true", - default=False, - ) # These can be removed once mkosi v15 is available in LTS distros and compatibility with <= v14 # is no longer needed in build infrastructure (e.g.: OBS). parser.add_argument( @@ -3272,19 +3294,18 @@ def create_argument_parser(action: type[argparse.Action], chdir: bool = True) -> def resolve_deps(images: Sequence[argparse.Namespace], include: Sequence[str]) -> list[argparse.Namespace]: graph = {config.image: config.dependencies for config in images} - if include: - if any((missing := i) not in graph for i in include): - die(f"No image found with name {missing}") + if any((missing := i) not in graph for i in include): + die(f"No image found with name {missing}") - deps = set() - queue = [*include] + deps = set() + queue = [*include] - while queue: - if (image := queue.pop(0)) not in deps: - deps.add(image) - queue.extend(graph[image]) + while queue: + if (image := queue.pop(0)) not in deps: + deps.add(image) + queue.extend(graph[image]) - images = [config for config in images if config.image in deps] + images = [config for config in images if config.image in deps] graph = {config.image: config.dependencies for config in images} @@ -3297,12 +3318,16 @@ def resolve_deps(images: Sequence[argparse.Namespace], include: Sequence[str]) - def parse_config(argv: Sequence[str] = (), *, resources: Path = Path("/")) -> tuple[Args, tuple[Config, ...]]: - # Compare inodes instead of paths so we can't get tricked by bind mounts and such. - namespace = argparse.Namespace() - defaults = argparse.Namespace() - parsed_includes: set[tuple[int, int]] = set() - immutable_settings: set[str] = set() - append = False + + class ParseContext: + # We keep two namespaces around, one for the settings specified on the CLI and one for the settings specified + # in configuration files. This is required to implement both [Match] support and the behavior where settings + # specified on the CLI always override settings specified in configuration files. + cli = argparse.Namespace() + config = argparse.Namespace() + # Compare inodes instead of paths so we can't get tricked by bind mounts and such. + includes: set[tuple[int, int]] = set() + immutable: set[str] = set() def expand_specifiers(text: str, path: Path) -> str: percent = False @@ -3323,17 +3348,26 @@ def expand_specifiers(text: str, path: Path) -> str: result += str(v) elif specifier := SPECIFIERS_LOOKUP_BY_CHAR.get(c): + specifierns = argparse.Namespace() + + # Some specifier methods might want to access the image name or directory mkosi was invoked in so + # let's make sure those are available. + setattr(specifierns, "image", getattr(ParseContext.config, "image", None)) + setattr(specifierns, "directory", ParseContext.cli.directory) + for d in specifier.depends: setting = SETTINGS_LOOKUP_BY_DEST[d] - if finalize_value(setting) is None: + if (v := finalize_value(setting)) is None: logging.warning( f"Setting {setting.name} which specifier '%{c}' in {text} depends on is not yet set, " "ignoring" ) break + + setattr(specifierns, d, v) else: - result += specifier.callback(namespace, path) + result += specifier.callback(specifierns, path) else: logging.warning(f"Unknown specifier '%{c}' found in {text}, ignoring") elif c == "%": @@ -3352,7 +3386,7 @@ def parse_new_includes() -> Iterator[None]: yield finally: # Parse any includes that were added after yielding. - for p in getattr(namespace, "include", []): + for p in getattr(ParseContext.cli, "include", []) + getattr(ParseContext.config, "include", []): for c in BUILTIN_CONFIGS: if p == Path(c): path = resources / c @@ -3362,10 +3396,10 @@ def parse_new_includes() -> Iterator[None]: st = path.stat() - if (st.st_dev, st.st_ino) in parsed_includes: + if (st.st_dev, st.st_ino) in ParseContext.includes: continue - parsed_includes.add((st.st_dev, st.st_ino)) + ParseContext.includes.add((st.st_dev, st.st_ino)) if any(p == Path(c) for c in BUILTIN_CONFIGS): _, [config] = parse_config(["--directory", "", "--include", os.fspath(path)]) @@ -3393,9 +3427,6 @@ def __call__( ) -> None: assert option_string is not None - if namespace.append != append: - return - if values is None and self.nargs == "?": values = self.const or "yes" @@ -3413,24 +3444,58 @@ def __call__( setattr(namespace, s.dest, s.parse(v, getattr(namespace, self.dest, None))) def finalize_value(setting: ConfigSetting) -> Optional[Any]: - if (v := getattr(namespace, setting.dest, None)) is not None: - return v + # If a value was specified on the CLI, it always takes priority. If the setting is a collection of values, we + # merge the value from the CLI with the value from the configuration, making sure that the value from the CLI + # always takes priority. + if ( + hasattr(ParseContext.cli, setting.dest) and + (v := getattr(ParseContext.cli, setting.dest)) is not None + ): + if isinstance(v, list): + return (getattr(ParseContext.config, setting.dest, None) or []) + v + elif isinstance(v, dict): + return (getattr(ParseContext.config, setting.dest, None) or {}) | v + elif isinstance(v, set): + return (getattr(ParseContext.config, setting.dest, None) or set()) | v + else: + return v - for d in setting.default_factory_depends: - finalize_value(SETTINGS_LOOKUP_BY_DEST[d]) + # If the setting was assigned the empty string on the CLI, we don't use any value configured in the + # configuration file. Additionally, if the setting is a collection of values, we won't use any default + # value either if the setting is set to the empty string on the command line. - # If the setting was assigned the empty string, we don't use any configured default value. - if not hasattr(namespace, setting.dest) and setting.dest in defaults: - default = getattr(defaults, setting.dest) - elif setting.default_factory: - default = setting.default_factory(namespace) - elif setting.default is None: + if ( + not hasattr(ParseContext.cli, setting.dest) and + hasattr(ParseContext.config, setting.dest) and + (v := getattr(ParseContext.config, setting.dest)) is not None + ): + return v + + if ( + (hasattr(ParseContext.cli, setting.dest) or hasattr(ParseContext.config, setting.dest)) and + isinstance(setting.parse(None, None), (dict, list, set)) + ): default = setting.parse(None, None) - else: + elif setting.default_factory: + # To determine default values, we need the final values of various settings in + # a namespace object, but we don't want to copy the final values into the config + # namespace object just yet so we create a new namespace object instead. + factoryns = argparse.Namespace( + **{d: finalize_value(SETTINGS_LOOKUP_BY_DEST[d]) for d in setting.default_factory_depends} + ) + + # Some default factory methods want to access the image name or directory mkosi + # was invoked in so let's make sure those are available. + setattr(factoryns, "image", getattr(ParseContext.config, "image", None)) + setattr(factoryns, "directory", ParseContext.cli.directory) + + default = setting.default_factory(factoryns) + elif setting.default is not None: default = setting.default + else: + default = setting.parse(None, None) - with parse_new_includes(): - setattr(namespace, setting.dest, default) + setattr(ParseContext.config, setting.dest, default) return default @@ -3502,7 +3567,7 @@ def match_config(path: Path) -> bool: return match_triggered is not False - def parse_config_one(path: Path, profiles: bool = False) -> bool: + def parse_config_one(path: Path, profiles: bool = False, local: bool = False) -> bool: s: Optional[ConfigSetting] # Make mypy happy extras = path.is_dir() @@ -3513,11 +3578,17 @@ def parse_config_one(path: Path, profiles: bool = False) -> bool: return False if extras: - if (path.parent / "mkosi.local.conf").exists(): + if local and (path.parent / "mkosi.local.conf").exists(): parse_config_one(path.parent / "mkosi.local.conf") + # Configuration from mkosi.local.conf should override other file based configuration but not the CLI + # itself so move the finalized values to the CLI namespace. + for s in SETTINGS: + if hasattr(ParseContext.config, s.dest): + setattr(ParseContext.cli, s.dest, finalize_value(s)) + delattr(ParseContext.config, s.dest) + for s in SETTINGS: - ns = defaults if s.path_default else namespace for f in s.paths: p = parse_path( f, @@ -3529,42 +3600,45 @@ def parse_config_one(path: Path, profiles: bool = False) -> bool: ) if p.exists(): setattr( - ns, + ParseContext.config, s.dest, - s.parse(p.read_text().rstrip("\n") if s.path_read_text else f, getattr(ns, s.dest, None)), + s.parse( + p.read_text().rstrip("\n") if s.path_read_text else f, + getattr(ParseContext.config, s.dest, None) + ), ) if path.exists(): logging.debug(f"Including configuration file {Path.cwd() / path}") - for section, k, v in parse_ini(path, only_sections={s.section for s in SETTINGS} | {"Preset"}): + for section, k, v in parse_ini(path, only_sections={s.section for s in SETTINGS}): if not k and not v: continue name = k.removeprefix("@") - ns = namespace if k == name else defaults + if name != k: + logging.warning(f"The '@' specifier is deprecated, please use {name} instead of {k}") if not (s := SETTINGS_LOOKUP_BY_NAME.get(name)): - die(f"Unknown setting {k}") - if name in immutable_settings: + die(f"Unknown setting {name}") + if name in ParseContext.immutable: die(f"Setting {name} cannot be modified anymore at this point") if section != s.section: - logging.warning(f"Setting {k} should be configured in [{s.section}], not [{section}].") + logging.warning(f"Setting {name} should be configured in [{s.section}], not [{section}].") if name != s.name: - canonical = s.name if k == name else f"@{s.name}" - logging.warning(f"Setting {k} is deprecated, please use {canonical} instead.") + logging.warning(f"Setting {name} is deprecated, please use {s.name} instead.") v = expand_specifiers(v, path) with parse_new_includes(): - setattr(ns, s.dest, s.parse(v, getattr(ns, s.dest, None))) + setattr(ParseContext.config, s.dest, s.parse(v, getattr(ParseContext.config, s.dest, None))) if profiles: finalize_value(SETTINGS_LOOKUP_BY_DEST["profile"]) - profile = getattr(namespace, "profile") - immutable_settings.add("Profile") + profile = getattr(ParseContext.config, "profile") + ParseContext.immutable.add("Profile") if profile: for p in (profile, f"{profile}.conf"): @@ -3574,7 +3648,7 @@ def parse_config_one(path: Path, profiles: bool = False) -> bool: else: die(f"Profile '{profile}' not found in mkosi.profiles/") - setattr(namespace, "profile", profile) + setattr(ParseContext.config, "profile", profile) with chdir(p if p.is_dir() else Path.cwd()): parse_config_one(p if p.is_file() else Path(".")) @@ -3587,11 +3661,6 @@ def parse_config_one(path: Path, profiles: bool = False) -> bool: return True - def finalize_values() -> None: - for s in SETTINGS: - finalize_value(s) - - images = [] argv = list(argv) # Make sure the verb command gets explicitly passed. Insert a -- before the positional verb argument @@ -3613,108 +3682,90 @@ def finalize_values() -> None: else: argv += ["--", "build"] - argparser = create_argument_parser(ConfigAction) - argparser.parse_args(argv, namespace) - cli_ns = copy.deepcopy(namespace) + # The "image" field does not directly map to a setting but is required + # to determine some default values for settings, so let's set it on the + # config namespace immediately so it's available. + setattr(ParseContext.config, "image", None) - args = load_args(namespace) + # First, we parse the command line arguments into a separate namespace. + argparser = create_argument_parser(ConfigAction) + argparser.parse_args(argv, ParseContext.cli) + args = load_args(ParseContext.cli) + # If --debug was passed, apply it as soon as possible. if ARG_DEBUG.get(): logging.getLogger().setLevel(logging.DEBUG) + # Do the same for help. if args.verb == Verb.help: - PagerHelpAction.__call__(None, argparser, namespace) # type: ignore + PagerHelpAction.__call__(None, argparser, ParseContext.cli) # type: ignore if not args.verb.needs_config(): return args, () - include = () + # One of the specifiers needs access to the directory so let's make sure it + # is available. + setattr(ParseContext.config, "directory", args.directory) + # Parse the global configuration unless the user explicitly asked us not to. if args.directory is not None: - parse_config_one(Path("."), profiles=True) - - finalize_value(SETTINGS_LOOKUP_BY_DEST["images"]) - include = getattr(namespace, "images") - immutable_settings.add("Images") - - d: Optional[Path] - for d in (Path("mkosi.images"), Path("mkosi.presets")): - if Path(d).exists(): - break - else: - d = None - - if d: - for p in sorted(d.iterdir()): - if not p.is_dir() and not p.suffix == ".conf": - continue - - name = p.name.removesuffix(".conf") - if not name: - die(f"{p} is not a valid image name") - - ns_copy = copy.deepcopy(namespace) - defaults_copy = copy.deepcopy(defaults) - parsed_includes_copy = copy.deepcopy(parsed_includes) - - setattr(namespace, "image", name) + parse_config_one(Path("."), profiles=True, local=True) - with chdir(p if p.is_dir() else Path.cwd()): - if not parse_config_one(p if p.is_file() else Path(".")): - continue + # After we've finished parsing the configuration, we'll have values in both + # namespaces (ParseContext.cli, ParseContext.config). To be able to parse the values from a + # single namespace, we merge the final values of each setting into one namespace. + for s in SETTINGS: + setattr(ParseContext.config, s.dest, finalize_value(s)) - finalize_values() - images += [namespace] + # Load the configuration for the main image. + config = load_config(ParseContext.config) - namespace = ns_copy - defaults = defaults_copy - parsed_includes = parsed_includes_copy + images = [] - if not images: - setattr(namespace, "image", None) - finalize_values() - images = [namespace] + if args.directory is not None and Path("mkosi.images").exists(): + # For the subimages in mkosi.images/, we want settings that are marked as + # "universal" to override whatever settings are specified in the subimage + # configuration files. We achieve this by making it appear like these settings + # were specified on the CLI by copying them to the CLI namespace. Any settings + # that are not marked as "universal" are deleted from the CLI namespace. + for s in SETTINGS: + if s.universal: + setattr(ParseContext.cli, s.dest, getattr(ParseContext.config, s.dest)) + elif hasattr(ParseContext.cli, s.dest): + delattr(ParseContext.cli, s.dest) - append = True + for p in sorted(Path("mkosi.images").iterdir()): + if not p.is_dir() and not p.suffix == ".conf": + continue - if args.append: - for ns in images: - ns.append = False - create_argument_parser(ConfigAction, chdir=False).parse_args(argv, ns) + name = p.name.removesuffix(".conf") + if not name: + die(f"{p} is not a valid image name") - for s in vars(cli_ns): - if s not in SETTINGS_LOOKUP_BY_DEST: - continue + ParseContext.config = argparse.Namespace() + setattr(ParseContext.config, "image", name) + setattr(ParseContext.config, "directory", args.directory) - if getattr(cli_ns, s) is None: - continue + # Allow subimage configuration to include everything again. + ParseContext.includes = set() - if isinstance(getattr(cli_ns, s), (list, tuple, dict)): - continue - - if any(getattr(config, s) == getattr(cli_ns, s) for config in images): - continue + with chdir(p if p.is_dir() else Path.cwd()): + if not parse_config_one(p if p.is_file() else Path("."), local=True): + continue - setting = SETTINGS_LOOKUP_BY_DEST[s].long - a = getattr(cli_ns, s) - die( - f"{setting}={a} was specified on the command line but is not allowed to be configured by any images.", - hint="Prefix the setting with '@' in the image configuration file to allow overriding it from the command line.", # noqa: E501 - ) + # Consolidate all settings into one namespace again. + for s in SETTINGS: + setattr(ParseContext.config, s.dest, finalize_value(s)) - if not images: - die("No images defined in mkosi.images/") + images += [ParseContext.config] - images = resolve_deps(images, include) + images = resolve_deps(images, config.dependencies) images = [load_config(ns) for ns in images] - return args, tuple(images) + return args, tuple(images + [config]) def load_credentials(args: argparse.Namespace) -> dict[str, str]: - if not args.verb.needs_credentials(): - return {} - creds = { "agetty.autologin": "root", "login.noauth": "yes", @@ -3854,6 +3905,9 @@ def load_args(args: argparse.Namespace) -> Args: def load_config(config: argparse.Namespace) -> Config: + # Make sure we don't modify the input namespace. + config = copy.deepcopy(config) + if config.build_dir: config.build_dir /= config.build_dir / f"{config.distribution}~{config.release}~{config.architecture}" if config.image: @@ -3862,8 +3916,10 @@ def load_config(config: argparse.Namespace) -> Config: if config.sign: config.checksum = True - config.credentials = load_credentials(config) - config.kernel_command_line_extra = load_kernel_command_line_extra(config) + if not config.image: + config.credentials = load_credentials(config) + config.kernel_command_line_extra = load_kernel_command_line_extra(config) + config.environment = load_environment(config) if config.overlay and not config.base_trees: @@ -3937,7 +3993,6 @@ def bold(s: Any) -> str: Profile: {none_to_none(config.profile)} Include: {line_join_list(config.include)} Initrd Include: {line_join_list(config.initrd_include)} - Images: {line_join_list(config.images)} Dependencies: {line_join_list(config.dependencies)} Minimum Version: {none_to_none(config.minimum_version)} Configure Scripts: {line_join_list(config.configure_scripts)} diff --git a/mkosi/resources/mkosi-initrd/mkosi.conf b/mkosi/resources/mkosi-initrd/mkosi.conf index 505f53072..ede214487 100644 --- a/mkosi/resources/mkosi-initrd/mkosi.conf +++ b/mkosi/resources/mkosi-initrd/mkosi.conf @@ -1,15 +1,15 @@ # SPDX-License-Identifier: LGPL-2.1-or-later [Output] -@Output=initrd -@Format=cpio +Output=initrd +Format=cpio ManifestFormat= [Content] BuildSources= Bootable=no MakeInitrd=yes -@CleanPackageMetadata=yes +CleanPackageMetadata=yes Packages= systemd # sine qua non udev @@ -34,7 +34,7 @@ RemoveFiles= /usr/lib/modules/*/System.map # Configure locale explicitly so that all other locale data is stripped on distros whose package manager supports it. -@Locale=C.UTF-8 +Locale=C.UTF-8 WithDocs=no # Make sure various core modules are always included in the initrd. diff --git a/mkosi/resources/mkosi-tools/mkosi.conf b/mkosi/resources/mkosi-tools/mkosi.conf index 8a5edd844..e26c886a9 100644 --- a/mkosi/resources/mkosi-tools/mkosi.conf +++ b/mkosi/resources/mkosi-tools/mkosi.conf @@ -2,7 +2,7 @@ [Output] Format=directory -@Output=mkosi.tools +Output=mkosi.tools ManifestFormat= [Content] diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index 5521f46a9..4ada324a2 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -297,26 +297,27 @@ Configuration is parsed in the following order: * `mkosi.conf` is parsed if it exists in the directory configured with `--directory=` or the current working directory if `--directory=` is not used. +* If a profile is defined, its configuration is parsed from the + `mkosi.profiles/` directory. * `mkosi.conf.d/` is parsed in the same directory if it exists. Each directory and each file with the `.conf` extension in `mkosi.conf.d/` is parsed. Any directory in `mkosi.conf.d` is parsed as if it were a regular top level directory. - -Note that if the same setting is configured twice, the later assignment -overrides the earlier assignment unless the setting is a list based -setting. Also note that before v16, we used to do the opposite, where -the earlier assignment would be used instead of later assignments. - -Settings that take a list of values are merged by appending the new -values to the previously configured values. Assigning the empty string -to such a setting removes all previously assigned values, and overrides -any configured default values as well. - -If a setting's name in the configuration file is prefixed with `@`, it -configures the default value used for that setting if no explicit -default value is set. This can be used to set custom default values in -configuration files that can still be overridden by specifying the -setting explicitly via the CLI. +* Subimages are parsed from the `mkosi.images` directory if it exists. + +Note that settings configured via the command line always override +settings configured via configuration files. If the same setting is +configured more than once via configuration files, later assignments +override earlier assignments except for settings that take a collection +of values. Also, settings read from `mkosi.local.conf` will override +settings from configuration files that are parsed later but not settings +specified on the CLI. + +Settings that take a collection of values are merged by appending the +new values to the previously configured values. Assigning the empty +string to such a setting removes all previously assigned values, and +overrides any configured default values as well. The values specified +on the CLI are appended after all the values from configuration files. To conditionally include configuration files, the `[Match]` section can be used. A `[Match]` section consists of individual conditions. @@ -331,9 +332,10 @@ exclamation second. Note that `[Match]` conditions compare against the current values of specific settings, and do not take into account changes made to the -setting in configuration files that have not been parsed yet. Also note -that matching against a setting and then changing its value afterwards -in a different config file may lead to unexpected results. +setting in configuration files that have not been parsed yet (settings +specified on the CLI are taken into account). Also note that matching +against a setting and then changing its value afterwards in a different +config file may lead to unexpected results. The `[Match]` section of a `mkosi.conf` file in a directory applies to the entire directory. If the conditions are not satisfied, the entire @@ -2422,19 +2424,62 @@ be recompiled. # Building multiple images If the `mkosi.images/` directory exists, mkosi will load individual -image configurations from it and build each of them. Image +subimage configurations from it and build each of them. Image configurations can be either directories containing mkosi configuration files or regular files with the `.conf` extension. When image configurations are found in `mkosi.images/`, mkosi will build -the configured images and all of their dependencies (or all of them if -no images were explicitly configured using `Images=`). To add -dependencies between images, the `Dependencies=` setting can be used. - -When images are defined, mkosi will first read the global configuration -(configuration outside of the `mkosi.images/` directory), followed by -the image specific configuration. This means that global configuration -takes precedence over image specific configuration. +the images specified in the `Dependencies=` setting of the main image +and all of their dependencies (or all of them if no images were +explicitly configured using `Dependencies=` in the main image +configuration). To add dependencies between subimages, the +`Dependencies=` setting can be used as well. Subimages are always built +before the main image. + +When images are defined, mkosi will first read the main image +configuration (configuration outside of the `mkosi.images/` directory), +followed by the image specific configuration. Several "universal" +settings apply to the main image and all its subimages and cannot be +configured separately in subimages. The following settings are universal +and cannot be configured in subimages (except for settings which take a +collection of values which can be extended in subimages but not +overridden): + +- `Profile=` +- `Distribution=` +- `Release=` +- `Architecture=` +- `Mirror=` +- `LocalMirror=` +- `RepositoryKeyCheck=` +- `Repositories=` +- `CacheOnly=` +- `PackageManagerTrees=` +- `OutputDirectory=` +- `WorkspaceDirectory=` +- `CacheDirectory=` +- `PackageCacheDirectory=` +- `BuildDirectory=` +- `ImageId=` +- `ImageVersion=` +- `SectorSize=` +- `RepartOffline=` +- `UseSubvolumes=` +- `PackageDirectories=` +- `SourceDateEpoch=` +- `VerityKey=` +- `VerityKeySource=` +- `VerityCertificate=` +- `ProxyUrl=` +- `ProxyExclude=` +- `ProxyPeerCertificate=` +- `ProxyClientCertificate=` +- `ProxyClientKey=` +- `Incremental=` +- `ExtraSearchPaths=` +- `Acl=` +- `ToolsTree=` +- `ToolsTreeCertificates=` Images can refer to outputs of images they depend on. Specifically, for the following options, mkosi will only check whether the inputs diff --git a/mkosi/util.py b/mkosi/util.py index 34c2c434b..8b1d1f199 100644 --- a/mkosi/util.py +++ b/mkosi/util.py @@ -182,6 +182,10 @@ def _generate_next_value_(name: str, start: int, count: int, last_values: Sequen def values(cls) -> list[str]: return list(s.replace("_", "-") for s in map(str, cls.__members__)) + @classmethod + def choices(cls) -> list[str]: + return [*cls.values(), ""] + @contextlib.contextmanager def umask(mask: int) -> Iterator[None]: diff --git a/tests/test_config.py b/tests/test_config.py index 712764622..28eb3c0e6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -23,7 +23,7 @@ parse_config, parse_ini, ) -from mkosi.distributions import Distribution +from mkosi.distributions import Distribution, detect_distribution from mkosi.util import chdir @@ -97,16 +97,20 @@ def test_parse_config(tmp_path: Path) -> None: (d / "mkosi.conf").write_text( """\ [Distribution] - - @Distribution = ubuntu - Architecture = arm64 + Distribution=ubuntu + Architecture=arm64 + Repositories=epel,epel-next [Content] Packages=abc + Environment=MY_KEY=MY_VALUE [Output] - @Format = cpio - ImageId = base + Format=cpio + ImageId=base + + [Host] + Credentials=my.cred=my.value """ ) @@ -120,44 +124,68 @@ def test_parse_config(tmp_path: Path) -> None: assert config.image_id == "base" with chdir(d): - _, [config] = parse_config(["--distribution", "fedora"]) + _, [config] = parse_config( + [ + "--distribution", "fedora", + "--environment", "MY_KEY=CLI_VALUE", + "--credential", "my.cred=cli.value", + "--repositories", "universe", + ] + ) - # mkosi.conf sets a default distribution, so the CLI should take priority. + # Values from the CLI should take priority. assert config.distribution == Distribution.fedora + assert config.environment["MY_KEY"] == "CLI_VALUE" + assert config.credentials["my.cred"] == "cli.value" + assert config.repositories == ["epel", "epel-next", "universe"] - # Any architecture set on the CLI is overridden by the config file, and we should complain loudly about that. - with chdir(d), pytest.raises(SystemExit): - _, [config] = parse_config(["--architecture", "x86-64"]) + with chdir(d): + _, [config] = parse_config( + [ + "--distribution", "", + "--environment", "", + "--credential", "", + "--repositories", "", + ] + ) + + # Empty values on the CLIs resets non-collection based settings to their defaults and collection based settings to + # empty collections. + assert config.distribution == detect_distribution()[0] + assert "MY_KEY" not in config.environment + assert "my.cred" not in config.credentials + assert config.repositories == [] (d / "mkosi.conf.d").mkdir() (d / "mkosi.conf.d/d1.conf").write_text( """\ [Distribution] - Distribution = debian - @Architecture = x86-64 + Distribution=debian [Content] - Packages = qed - def + Packages=qed + def [Output] - ImageId = 00-dropin - ImageVersion = 0 + ImageId=00-dropin + ImageVersion=0 + @Output=abc """ ) with chdir(d): - _, [config] = parse_config() + _, [config] = parse_config(["--package", "last"]) # Setting a value explicitly in a dropin should override the default from mkosi.conf. assert config.distribution == Distribution.debian - # Setting a default in a dropin should be ignored since mkosi.conf sets the architecture explicitly. - assert config.architecture == Architecture.arm64 - # Lists should be merged by appending the new values to the existing values. - assert config.packages == ["abc", "qed", "def"] + # Lists should be merged by appending the new values to the existing values. Any values from the CLI should be + # appended to the values from the configuration files. + assert config.packages == ["abc", "qed", "def", "last"] assert config.output_format == OutputFormat.cpio assert config.image_id == "00-dropin" assert config.image_version == "0" + # '@' specifier should be automatically dropped. + assert config.output == "abc" (d / "mkosi.version").write_text("1.2.3") @@ -221,6 +249,38 @@ def test_parse_config(tmp_path: Path) -> None: assert config.bootable == ConfigFeature.enabled assert config.split_artifacts is False + (d / "mkosi.images").mkdir() + + for n in ("one", "two"): + (d / "mkosi.images" / f"{n}.conf").write_text( + f"""\ + [Distribution] + Release=bla + + [Content] + Packages={n} + """ + ) + + with chdir(d): + _, [one, two, config] = parse_config(["--package", "qed", "--build-package", "def"]) + + # Universal settings should always come from the main image. + assert one.distribution == config.distribution + assert two.distribution == config.distribution + assert one.release == config.release + assert two.release == config.release + + # Non-universal settings should not be passed to the subimages. + assert one.packages == ["one"] + assert two.packages == ["two"] + assert one.build_packages == [] + assert two.build_packages == [] + + # But should apply to the main image of course. + assert config.packages == ["qed"] + assert config.build_packages == ["def"] + def test_parse_includes_once(tmp_path: Path) -> None: d = tmp_path @@ -254,9 +314,9 @@ def test_parse_includes_once(tmp_path: Path) -> None: ) with chdir(d): - _, [one, two] = parse_config([]) - assert one.build_packages == ["abc", "def"] - assert two.build_packages == ["abc", "def"] + _, [one, two, config] = parse_config([]) + assert one.build_packages == ["def"] + assert two.build_packages == ["def"] def test_profiles(tmp_path: Path) -> None: @@ -302,15 +362,19 @@ def test_override_default(tmp_path: Path) -> None: (d / "mkosi.conf").write_text( """\ + [Content] + Environment=MY_KEY=MY_VALUE + [Host] - @ToolsTree=default + ToolsTree=default """ ) with chdir(d): - _, [config] = parse_config(["--tools-tree", ""]) + _, [config] = parse_config(["--tools-tree", "", "--environment", ""]) assert config.tools_tree is None + assert "MY_KEY" not in config.environment def test_local_config(tmp_path: Path) -> None: @@ -320,6 +384,9 @@ def test_local_config(tmp_path: Path) -> None: """\ [Distribution] Distribution=debian + + [Content] + WithTests=yes """ ) @@ -332,13 +399,24 @@ def test_local_config(tmp_path: Path) -> None: """\ [Distribution] Distribution=fedora + + [Content] + WithTests=no """ ) with chdir(d): _, [config] = parse_config() + # Local config should take precedence over non-local config. + assert config.distribution == Distribution.debian + assert config.with_tests + + with chdir(d): + _, [config] = parse_config(["--distribution", "fedora", "-T"]) + assert config.distribution == Distribution.fedora + assert not config.with_tests def test_parse_load_verb(tmp_path: Path) -> None: diff --git a/tests/test_json.py b/tests/test_json.py index e5427249a..60b3a2a9a 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -40,7 +40,6 @@ def test_args(path: Optional[Path]) -> None: dump = textwrap.dedent( f"""\ {{ - "Append": true, "AutoBump": false, "Cmdline": [ "foo", @@ -62,7 +61,6 @@ def test_args(path: Optional[Path]) -> None: ) args = Args( - append = True, auto_bump = False, cmdline = ["foo", "bar"], debug = False, @@ -145,10 +143,6 @@ def test_config() -> None: "Image": "default", "ImageId": "myimage", "ImageVersion": "5", - "Images": [ - "default", - "initrd" - ], "Include": [], "Incremental": false, "InitrdInclude": [ @@ -396,7 +390,6 @@ def test_config() -> None: image="default", image_id="myimage", image_version="5", - images=["default", "initrd"], include=[], incremental=False, initrd_include=[Path("/foo/bar"),],