refactor: update runtime directory structure (#400)

* umu_consts: add enum for global flocks

* umu: update flocks to use enum

* __init__: update runtime metadata

* umu_run: add support for solving correct runtime

* umu_runtime: refactor to reference container runtime subdir

* umu: update tests to handle parameter change

* umu_test: update tests to handle new parameter

* umu_runtime: refactor to not refer to constant

* umu_test: update tests

* tests: add e2e test for using obsolete protons

* tests: fix unused tmp

* umu_runtime: refactor to not refer to constant

* umu_runtime: require a path for umu-shim

- We don't want any files besides our locks to be within the top-level

* umu_test: remove test for default shim path

* umu_runtime: fix unused constant

* tests: update test_update.sh

* tests: set RUNTIMEPATH explicitly when using steamrt3-based protons

* umu_run: resolve RUNTIMEPATH when configuring environment

* Revert "umu_run: resolve RUNTIMEPATH when configuring environment"

This reverts commit d9b98a8.

* umu_run: update STEAM_COMPAT_TOOL_PATHS

* umu_run: create all segments up to the runtime subdir

* umu_run: set RUNTIMEPATH

* umu: set RUNTIMEPATH for environment tests

* umu_test: fix assertions for STEAM_COMPAT_TOOL_PATHS

* umu_run: update function parameters

* umu_test: fix creation of umu.lock

* umu_run: fix unused import

.github/workflows/e2e.yml

@@ -51,6 +51,11 @@ jobs:
           source .venv/bin/activate
           sh tests/test_install.sh
           rm -rf "$HOME/.local/share/umu" "$HOME/Games/umu" "$HOME/.local/share/Steam/compatibilitytools.d"
+      - name: Test obsolete steamrt install
+        run: |
+          source .venv/bin/activate
+          sh tests/test_install_obsolete.sh
+          rm -rf "$HOME/.local/share/umu" "$HOME/Games/umu" "$HOME/.local/share/Steam/compatibilitytools.d"
       - name: Test steamrt update
         run: |
           source .venv/bin/activate

tests/test_config.sh

@@ -19,5 +19,5 @@ store = 'gog'
 " >> "$tmp"
 
 
-UMU_LOG=debug GAMEID=umu-1141086411 STORE=gog "$PWD/.venv/bin/python" "$HOME/.local/bin/umu-run" --config "$tmp" 2> /tmp/umu-log.txt && grep -E "INFO: Non-steam game Silent Hill 4: The Room \(umu-1141086411\)" /tmp/umu-log.txt
+RUNTIMEPATH=steamrt3 UMU_LOG=debug GAMEID=umu-1141086411 STORE=gog "$PWD/.venv/bin/python" "$HOME/.local/bin/umu-run" --config "$tmp" 2> /tmp/umu-log.txt && grep -E "INFO: Non-steam game Silent Hill 4: The Room \(umu-1141086411\)" /tmp/umu-log.txt
 # Run the 'game' using UMU-Proton9.0-3.2 and ensure the protonfixes module finds its fix in umu-database.csv

tests/test_install_obsolete.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+
+mkdir -p "$HOME/.local/share/Steam/compatibilitytools.d"
+curl -LJO "https://github.com/GloriousEggroll/proton-ge-custom/releases/download/GE-Proton7-55/GE-Proton7-55.tar.gz"
+tar xaf GE-Proton7-55.tar.gz
+mv GE-Proton7-55 "$HOME/.local/share/Steam/compatibilitytools.d"
+
+UMU_LOG=debug PROTONPATH=GE-Proton7-55 "$HOME/.local/bin/umu-run" wineboot -u

tests/test_offline.sh

@@ -13,20 +13,20 @@ url=$(curl -L "https://api.github.com/repos/Open-Wine-Components/umu-proton/rele
 # Download Proton
 curl -LJO "$url"
 
-mkdir -p "$HOME"/.local/share/Steam/compatibilitytools.d "$HOME"/.local/share/umu "$HOME"/Games/umu
+mkdir -p "$HOME"/.local/share/Steam/compatibilitytools.d "$HOME"/.local/share/umu/steamrt3 "$HOME"/Games/umu
 
 # Extract the archives
 tar xaf "$name" -C "$HOME"/.local/share/Steam/compatibilitytools.d
 tar xaf SteamLinuxRuntime_sniper.tar.xz
 
-cp -a SteamLinuxRuntime_sniper/* "$HOME"/.local/share/umu
-mv "$HOME"/.local/share/umu/_v2-entry-point "$HOME"/.local/share/umu/umu
+cp -a SteamLinuxRuntime_sniper/* "$HOME"/.local/share/umu/steamrt3
+mv "$HOME"/.local/share/umu/steamrt3/_v2-entry-point "$HOME"/.local/share/umu/steamrt3/umu
 
 # Run offline using bwrap
 # TODO: Figure out why the command exits with a 127 when offline. The point
 # is that we're able to enter the container and we do not crash. For now,
 # just query a string that shows that session was offline
-UMU_LOG=debug GAMEID=umu-0 bwrap --unshare-net --bind / / --dev /dev --bind "$HOME" "$HOME" -- "$HOME/.local/bin/umu-run" wineboot -u 2> "$tmp"
+RUNTIMEPATH=steamrt3 UMU_LOG=debug GAMEID=umu-0 bwrap --unshare-net --bind / / --dev /dev --bind "$HOME" "$HOME" -- "$HOME/.local/bin/umu-run" wineboot -u 2> "$tmp"
 
 # Check if we exited. If we logged this statement then there were no errors
 # before entering the container

tests/test_resume.sh

@@ -9,5 +9,5 @@ mkdir -p "$HOME"/.cache/umu
 # Move to our cache so it can be picked up then resumed.
 # Note: Must include the *.parts extension
 mv SteamLinuxRuntime_sniper.tar.xz "$HOME"/.cache/umu/SteamLinuxRuntime_sniper.tar.xz."$id".parts
-UMU_LOG=debug GAMEID=umu-0 "$HOME/.local/bin/umu-run" wineboot -u 2> "$tmp"
+RUNTIMEPATH=steamrt3 UMU_LOG=debug GAMEID=umu-0 "$HOME/.local/bin/umu-run" wineboot -u 2> "$tmp"
 grep "resuming" "$tmp" && grep "exited with wait status" "$tmp"

tests/test_update.sh

@@ -1,13 +1,13 @@
 #!/usr/bin/env sh
 
-mkdir -p "$HOME/.local/share/umu"
+mkdir -p "$HOME/.local/share/umu/steamrt3"
 
 curl -LJO "https://repo.steampowered.com/steamrt3/images/0.20240916.101795/SteamLinuxRuntime_sniper.tar.xz"
 tar xaf SteamLinuxRuntime_sniper.tar.xz
-mv SteamLinuxRuntime_sniper/* "$HOME/.local/share/umu"
-mv "$HOME/.local/share/umu/_v2-entry-point" "$HOME/.local/share/umu/umu"
-echo "$@" > "$HOME/.local/share/umu/umu-shim" && chmod 700 "$HOME/.local/share/umu/umu-shim"
+mv SteamLinuxRuntime_sniper/* "$HOME/.local/share/umu/steamrt3"
+mv "$HOME/.local/share/umu/steamrt3/_v2-entry-point" "$HOME/.local/share/umu/steamrt3/umu"
+echo "$@" > "$HOME/.local/share/umu/steamrt3/umu-shim" && chmod 700 "$HOME/.local/share/umu/steamrt3/umu-shim"
 
 # Perform a preflight step, where we ensure everything is in order and create '$HOME/.local/share/umu/var'
 # Afterwards, run a 2nd time to perform the runtime update and ensure '$HOME/.local/share/umu/var' is removed
-UMU_LOG=debug GAMEID=umu-0 UMU_RUNTIME_UPDATE=0 "$HOME/.local/bin/umu-run" wineboot -u && UMU_LOG=debug GAMEID=umu-0 "$HOME/.local/bin/umu-run" wineboot -u
+UMU_LOG=debug GAMEID=umu-0 UMU_RUNTIME_UPDATE=0 "$HOME/.local/bin/umu-run" wineboot -u && RUNTIMEPATH=steamrt3 UMU_LOG=debug GAMEID=umu-0 "$HOME/.local/bin/umu-run" wineboot -u

umu/__init__.py

@@ -1,3 +1,6 @@
 __version__ = "1.2.5"  # noqa: D104
-__runtime_versions__ = (("sniper", "steamrt3"), ("soldier", "steamrt2"))
+__runtime_versions__ = (
+    ("sniper", "steamrt3", "1628350"),
+    ("soldier", "steamrt2", "1391110"),
+)
 __runtime_version__ = __runtime_versions__[0]

umu/umu_consts.py

@@ -36,6 +36,14 @@ class GamescopeAtom(Enum):
     BaselayerAppId = "GAMESCOPECTRL_BASELAYER_APPID"
 
 
+class FileLock(Enum):
+    """Files placed with an exclusive lock via flock(2)."""
+
+    Runtime = "umu.lock"  # UMU_RUNTIME lock
+    Compat = "compatibilitytools.d.lock"  # PROTONPATH lock
+    Prefix = "pfx.lock"  # WINEPREFIX lock
+
+
 STEAM_COMPAT: Path = Path.home().joinpath(
     ".local", "share", "Steam", "compatibilitytools.d"
 )

umu/umu_proton.py

@@ -19,7 +19,14 @@
 from urllib3.response import BaseHTTPResponse
 
 from umu.umu_bspatch import Content, ContentContainer, CustomPatcher
-from umu.umu_consts import STEAM_COMPAT, UMU_CACHE, UMU_COMPAT, UMU_LOCAL, HTTPMethod
+from umu.umu_consts import (
+    STEAM_COMPAT,
+    UMU_CACHE,
+    UMU_COMPAT,
+    UMU_LOCAL,
+    FileLock,
+    HTTPMethod,
+)
 from umu.umu_log import log
 from umu.umu_util import (
     extract_tarfile,
@@ -366,7 +373,7 @@ def _get_latest(
     proton: str
     # Name of the Proton version, which is either UMU-Proton or GE-Proton
     version: str = ProtonVersion.UMU.value
-    lockfile: str = f"{UMU_LOCAL}/compatibilitytools.d.lock"
+    lock: str = f"{UMU_LOCAL}/{FileLock.Compat.value}"
     latest_candidates: set[str]
 
     if not assets:
@@ -391,18 +398,15 @@ def _get_latest(
 
     # Use the latest UMU/GE-Proton
     try:
-        log.debug("Acquiring file lock '%s'...", lockfile)
-        with unix_flock(lockfile):
+        log.debug("Acquiring file lock '%s'...", lock)
+        with unix_flock(lock):
             # Once acquiring the lock check if Proton hasn't been installed
             if steam_compat.joinpath(proton).is_dir():
                 raise FileExistsError
-
             if umu_compat.joinpath(version).is_dir():
                 raise FileExistsError
-
             # Download the archive to a temporary directory
             _fetch_proton(env, session_caches, assets, session_pools)
-
             # Extract the archive then move the directory
             _install_proton(tarball, session_caches, compat_tools)
     except (ValueError, KeyboardInterrupt, HTTPError) as e:
@@ -494,7 +498,7 @@ def _get_delta(
         "GE-Latest" if os.environ.get("PROTONPATH") == "GE-Latest" else "UMU-Latest"
     )
     proton: Path = umu_compat.joinpath(version)
-    lockfile: str = f"{UMU_LOCAL}/compatibilitytools.d.lock"
+    lock: str = f"{UMU_LOCAL}/{FileLock.Compat.value}"
     cbor: ContentContainer
 
     if not assets:
@@ -521,13 +525,12 @@ def _get_delta(
         log.exception(e)
         return None
 
-    log.debug("Acquiring lock '%s'", lockfile)
-    with unix_flock(lockfile):
+    log.debug("Acquiring lock '%s'", lock)
+    with unix_flock(lock):
         tarball, _ = assets[1]
         build: str = tarball.removesuffix(".tar.gz")
         buildid: Path = umu_compat.joinpath(version, "compatibilitytool.vdf")
-
-        log.debug("Acquired lock '%s'", lockfile)
+        log.debug("Acquired lock '%s'", lock)
 
         # Check if we're up to date by doing a simple file check
         # Avoids the cost of creating threads and memory-mapped IO

umu/umu_run.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 import threading
 import time
@@ -10,6 +11,7 @@
 from contextlib import suppress
 from ctypes import CDLL, c_int, c_ulong
 from errno import ENETUNREACH
+from itertools import chain
 from zipfile import Path as ZipPath
 
 try:
@@ -35,13 +37,14 @@
 from Xlib.protocol.rq import Event
 from Xlib.xobject.drawable import Window
 
-from umu import __runtime_version__, __version__
+from umu import __runtime_versions__, __version__
 from umu.umu_consts import (
     PR_SET_CHILD_SUBREAPER,
     PROTON_VERBS,
     STEAM_COMPAT,
     STEAM_WINDOW_ID,
     UMU_LOCAL,
+    FileLock,
     GamescopeAtom,
 )
 from umu.umu_log import log
@@ -61,6 +64,8 @@
 
 NET_RETRIES = 1
 
+RuntimeVersion = tuple[str, str, str]
+
 
 def setup_pfx(path: str) -> None:
     """Prepare a Proton compatible WINE prefix."""
@@ -234,7 +239,9 @@ def set_env(
     env["PROTONPATH"] = str(protonpath)
     env["STEAM_COMPAT_DATA_PATH"] = env["WINEPREFIX"]
     env["STEAM_COMPAT_SHADER_PATH"] = f"{env['STEAM_COMPAT_DATA_PATH']}/shadercache"
-    env["STEAM_COMPAT_TOOL_PATHS"] = f"{env['PROTONPATH']}:{UMU_LOCAL}"
+    env["STEAM_COMPAT_TOOL_PATHS"] = (
+        f"{env['PROTONPATH']}:{UMU_LOCAL}/{os.environ['RUNTIMEPATH']}"
+    )
     env["STEAM_COMPAT_MOUNTS"] = env["STEAM_COMPAT_TOOL_PATHS"]
 
     # Zenity
@@ -253,6 +260,7 @@ def set_env(
     env["UMU_NO_RUNTIME"] = os.environ.get("UMU_NO_RUNTIME") or ""
     env["UMU_RUNTIME_UPDATE"] = os.environ.get("UMU_RUNTIME_UPDATE") or ""
     env["UMU_NO_PROTON"] = os.environ.get("UMU_NO_PROTON") or ""
+    env["RUNTIMEPATH"] = f"{UMU_LOCAL}/{os.environ['RUNTIMEPATH']}"
 
     return env
 
@@ -694,6 +702,69 @@ def run_command(command: tuple[Path | str, ...]) -> int:
     return ret
 
 
+def resolve_umu_version(runtimes: tuple[RuntimeVersion, ...]) -> RuntimeVersion | None:
+    """Resolve the required runtime of a compatibility tool."""
+    version: tuple[str, str, str] | None = None
+
+    if os.environ.get("RUNTIMEPATH") in set(chain.from_iterable(runtimes)):
+        # Skip the parsing and trust the client
+        log.debug("RUNTIMEPATH is codename, skipping version resolution")
+        return next(
+            member for member in runtimes if os.environ["RUNTIMEPATH"] in member
+        )
+
+    if not os.environ.get("PROTONPATH"):
+        log.debug("PROTONPATH unset, defaulting to '%s'", runtimes[0][1])
+        return runtimes[0]
+
+    # Default to latest runtime for codenames
+    if os.environ.get("PROTONPATH") in {"GE-Proton", "GE-Latest", "UMU-Latest"}:
+        log.debug("PROTONPATH is codename, defaulting to '%s'", runtimes[0][1])
+        return runtimes[0]
+
+    # Default to latest runtime for native Linux executables
+    if os.environ.get("UMU_NO_PROTON"):
+        log.debug("UMU_NO_PROTON set, defaulting to '%s'", runtimes[0][1])
+        return runtimes[0]
+
+    # Solve the required runtime for PROTONPATH
+    log.debug("PROTONPATH set, resolving its required runtime")
+    path: Path = STEAM_COMPAT.joinpath(os.environ.get("PROTONPATH", ""))
+    if os.environ.get("PROTONPATH") and path.is_dir():
+        os.environ["PROTONPATH"] = str(STEAM_COMPAT.joinpath(os.environ["PROTONPATH"]))
+
+    path = Path(os.environ["PROTONPATH"], "toolmanifest.vdf").resolve()
+    if path.is_file():
+        version = get_umu_version_from_manifest(path, runtimes)
+
+    return version
+
+
+def get_umu_version_from_manifest(
+    path: Path, runtimes: tuple[RuntimeVersion, ...]
+) -> RuntimeVersion | None:
+    """Find the required runtime from a compatibility tool's configuration file."""
+    key: str = "require_tool_appid"
+    appids: set[str] = {member[2] for member in runtimes}
+    appid: str = ""
+
+    with path.open(mode="r", encoding="utf-8") as file:
+        for line in file:
+            if key not in line:
+                continue
+            if match := re.search(r'"require_tool_appid"\s+"(\d+)', line):
+                appid = match.group(1)
+                break
+
+    if not appid:
+        return None
+
+    if appid not in appids:
+        return None
+
+    return next(member for member in runtimes if appid in member)
+
+
 def umu_run(args: Namespace | tuple[str, list[str]]) -> int:
     """Prepare and run an executable within the Steam Runtime.
 
@@ -729,9 +800,11 @@ def umu_run(args: Namespace | tuple[str, list[str]]) -> int:
         "UMU_NO_RUNTIME": "",
         "UMU_RUNTIME_UPDATE": "",
         "UMU_NO_PROTON": "",
+        "RUNTIMEPATH": "",
     }
     opts: list[str] = []
     prereq: bool = False
+    version: RuntimeVersion | None = None
     root: Traversable
 
     try:
@@ -781,6 +854,15 @@ def umu_run(args: Namespace | tuple[str, list[str]]) -> int:
         )
         raise RuntimeError(err)
 
+    # Resolve the runtime version for PROTONPATH
+    version = resolve_umu_version(__runtime_versions__)
+    if not version:
+        err: str = (
+            f"Failed to match '{os.environ.get('PROTONPATH')}' with a container runtime"
+        )
+        raise ValueError(err)
+    os.environ["RUNTIMEPATH"] = version[1]
+
     # Opt to use the system's native CA bundle rather than certifi's
     with suppress(ModuleNotFoundError):
         import truststore
@@ -796,13 +878,10 @@ def umu_run(args: Namespace | tuple[str, list[str]]) -> int:
         ThreadPoolExecutor() as thread_pool,
         PoolManager(timeout=timeout, retries=retries) as http_pool,
     ):
-        session_pools: tuple[ThreadPoolExecutor, PoolManager] = (
-            thread_pool,
-            http_pool,
-        )
+        session_pools: tuple[ThreadPoolExecutor, PoolManager] = (thread_pool, http_pool)
         # Setup the launcher and runtime files
         future: Future = thread_pool.submit(
-            setup_umu, root, UMU_LOCAL, __runtime_version__, session_pools
+            setup_umu, root, UMU_LOCAL / version[1], version, session_pools
         )
 
         if isinstance(args, Namespace):
@@ -811,10 +890,10 @@ def umu_run(args: Namespace | tuple[str, list[str]]) -> int:
             opts = args[1]  # Reference the executable options
             check_env(env, session_pools)
 
-        UMU_LOCAL.mkdir(parents=True, exist_ok=True)
+        UMU_LOCAL.joinpath(version[1]).mkdir(parents=True, exist_ok=True)
 
         # Prepare the prefix
-        with unix_flock(f"{UMU_LOCAL}/pfx.lock"):
+        with unix_flock(f"{UMU_LOCAL}/{FileLock.Prefix.value}"):
             setup_pfx(env["WINEPREFIX"])
 
         # Configure the environment
@@ -844,7 +923,7 @@ def umu_run(args: Namespace | tuple[str, list[str]]) -> int:
         sys.exit(1)
 
     # Build the command
-    command: tuple[Path | str, ...] = build_command(env, UMU_LOCAL, opts)
+    command: tuple[Path | str, ...] = build_command(env, UMU_LOCAL / version[1], opts)
     log.debug("%s", command)
 
     # Run the command