diff --git a/exasol/exaslpm/model/package_file_config.py b/exasol/exaslpm/model/package_file_config.py index b09be9e..431306e 100644 --- a/exasol/exaslpm/model/package_file_config.py +++ b/exasol/exaslpm/model/package_file_config.py @@ -1,8 +1,3 @@ -import pathlib -from enum import Enum -from pathlib import Path -from typing import List - from pydantic import ( BaseModel, field_validator, diff --git a/exasol/exaslpm/pkg_mgmt/cmd_executor.py b/exasol/exaslpm/pkg_mgmt/cmd_executor.py new file mode 100644 index 0000000..7054777 --- /dev/null +++ b/exasol/exaslpm/pkg_mgmt/cmd_executor.py @@ -0,0 +1,100 @@ +import subprocess +import sys +from collections.abc import ( + Callable, + Iterator, +) +from typing import Protocol + + +class CommandLogger(Protocol): + def info(self, msg: str) -> None: ... + def warn(self, msg: str) -> None: ... + def error(self, msg: str) -> None: ... + + +class StdLogger: + def info(self, msg: str) -> None: + self._last_msg = msg + sys.stdout.write(msg) + + def warn(self, msg: str) -> None: + self._last_msg = msg + sys.stderr.write(msg) + + def error(self, msg: str) -> None: + self._last_msg = msg + sys.stderr.write(msg) + + def get_last_msg(self) -> str: + return self._last_msg + + +class CommandResult: + def __init__( + self, + logger: CommandLogger, + fn_ret_code: Callable[[], int], + stdout: Iterator[str], + stderr: Iterator[str], + ): + self._log = logger + self._fn_return_code = fn_ret_code # a lambda to subprocess.open.wait + self._stdout = stdout + self._stderr = stderr + + def return_code(self): + return self._fn_return_code() + + def itr_stdout(self) -> Iterator[str]: + return self._stdout + + def itr_stderr(self) -> Iterator[str]: + return self._stderr + + def consume_results( + self, + consume_stdout: Callable[[str | bytes], None], + consume_stderr: Callable[[str | bytes], None], + ): + + def pick_next(out_stream, callback) -> bool: + try: + _val = next(out_stream) + callback(_val) + except StopIteration: + return False + return True + + # Read from _stdout and _stderr simultaneously + stdout_continue = True + stderr_continue = True + while stdout_continue or stderr_continue: + if stdout_continue: + stdout_continue = pick_next(self._stdout, consume_stdout) + if stderr_continue: + stderr_continue = pick_next(self._stderr, consume_stderr) + return self.return_code() + + def print_results(self): + ret_code = self.consume_results(self._log.info, self._log.error) + self._log.info(f"Return Code: {ret_code}") + + +class CommandExecutor: + def __init__(self, logger: CommandLogger): + self._log = logger + + def execute(self, cmd_strs: list[str]) -> CommandResult: + cmd_str = " ".join(cmd_strs) + self._log.info(f"Executing: {cmd_str}") + + sub_process = subprocess.Popen( + cmd_strs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + return CommandResult( + self._log, + fn_ret_code=lambda: sub_process.wait(), + stdout=iter(sub_process.stdout), + stderr=iter(sub_process.stderr), + ) diff --git a/exasol/exaslpm/pkg_mgmt/install_apt.py b/exasol/exaslpm/pkg_mgmt/install_apt.py new file mode 100644 index 0000000..156bef6 --- /dev/null +++ b/exasol/exaslpm/pkg_mgmt/install_apt.py @@ -0,0 +1,69 @@ +from exasol.exaslpm.model.package_file_config import AptPackages +from exasol.exaslpm.pkg_mgmt.cmd_executor import ( + CommandExecutor, + CommandLogger, +) + + +def prepare_update_command() -> list[str]: + update_cmd = ["apt-get", "-y", "update"] + return update_cmd + + +def prepare_clean_cmd() -> list[str]: + clean_cmd = ["apt-get", "-y", "clean"] + return clean_cmd + + +def prepare_autoremove_cmd() -> list[str]: + autoremove_cmd = ["apt-get", "-y", "autoremove"] + return autoremove_cmd + + +def prepare_ldconfig_cmd() -> list[str]: + ldconfig_cmd = ["ldconfig"] + return ldconfig_cmd + + +def prepare_locale_cmd() -> list[str]: + locale_cmd = ["locale-gen", "&&", "update-locale", "LANG=en_US.UTF8"] + return locale_cmd + + +def prepare_install_cmd(apt_packages: AptPackages) -> list[str]: + install_cmd = ["apt-get", "install", "-V", "-y", "--no-install-recommends"] + if apt_packages.packages is not None: + for package in apt_packages.packages: + install_cmd.append(f"{package.name}={package.version}") + return install_cmd + + +def install_via_apt( + apt_packages: AptPackages, executor: CommandExecutor, log: CommandLogger +): + if len(apt_packages.packages) > 0: + update_cmd = prepare_update_command() + cmd_res = executor.execute(update_cmd) + cmd_res.print_results() + + install_cmd = prepare_install_cmd(apt_packages) + cmd_res = executor.execute(install_cmd) + cmd_res.print_results() + + clean_cmd = prepare_clean_cmd() + cmd_res = executor.execute(clean_cmd) + cmd_res.print_results() + + autoremove_cmd = prepare_autoremove_cmd() + cmd_res = executor.execute(autoremove_cmd) + cmd_res.print_results() + + locale_cmd = prepare_locale_cmd() + cmd_res = executor.execute(locale_cmd) + cmd_res.print_results() + + ldconfig_cmd = prepare_ldconfig_cmd() + cmd_res = executor.execute(ldconfig_cmd) + cmd_res.print_results() + else: + log.error("Got an empty list of AptPackages") diff --git a/exasol/exaslpm/pkg_mgmt/install_packages.py b/exasol/exaslpm/pkg_mgmt/install_packages.py index dccaea7..c241620 100644 --- a/exasol/exaslpm/pkg_mgmt/install_packages.py +++ b/exasol/exaslpm/pkg_mgmt/install_packages.py @@ -1,6 +1,26 @@ import pathlib -import click +import yaml + +from exasol.exaslpm.model.package_file_config import ( + PackageFile, + Phase, +) +from exasol.exaslpm.pkg_mgmt.cmd_executor import ( + CommandExecutor, + StdLogger, +) +from exasol.exaslpm.pkg_mgmt.install_apt import install_via_apt + + +def parse_package_file( + package_file: PackageFile, phase_name: str, build_step_name: str +) -> Phase: + build_steps = package_file.build_steps + build_step = build_steps[build_step_name] + phases = build_step.phases + phase = phases[phase_name] + return phase def package_install( @@ -11,6 +31,7 @@ def package_install( conda_binary: pathlib.Path, r_binary: pathlib.Path, ): + """ click.echo( f"Phase: {phase}, \ Package File: {package_file}, \ @@ -19,6 +40,15 @@ def package_install( Conda Binary: {conda_binary}, \ R Binary: {r_binary}", ) + """ package_content = package_file.read_text() - click.echo(package_content) + try: + logger = StdLogger() + yaml_data = yaml.safe_load(package_content) + pkg_file = PackageFile.model_validate(yaml_data) + single_phase = parse_package_file(pkg_file, phase, build_step) + if single_phase.apt is not None: + install_via_apt(single_phase.apt, CommandExecutor(logger), logger) + except ValueError: + print("Error parsing package file") diff --git a/poetry.lock b/poetry.lock index f38ac43..50cd84c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -2008,7 +2008,7 @@ version = "6.0.3" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, @@ -2663,4 +2663,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.15" -content-hash = "68951365dc4d5c4a8047c692904943526862853504579e50d98fb7d07fd2a17b" +content-hash = "c4f669f8777ccba8c3136782d5c0ac109b3d569de2e814d3fd47f31da1c39270" diff --git a/pyproject.toml b/pyproject.toml index 44ced0f..1ff37d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ homepage = "https://github.com/exasol/script-languages-package-management" [tool.poetry.dependencies] click = ">=8.3.0, <9.0.0" pydantic="^2.10.2" +pyyaml = "^6.0.3" [tool.poetry.group.dev.dependencies] exasol-toolbox = ">=1.6.0, <2.0.0" @@ -31,6 +32,9 @@ pyinstaller = ">=6.15.0, <7.0.0" types-pyinstaller = ">=6.15.0.20250822, <7.0.0.0" docker = { version = ">=7.1.0,<8.0.0", markers = "sys_platform != 'win32'" } +[tool.poetry.scripts] +exaslpm = 'exasol.exaslpm.main:main' + [build-system] requires = ["poetry-core>=2.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/test/resources/package_file_01.yaml b/test/resources/package_file_01.yaml new file mode 100644 index 0000000..2e0a97c --- /dev/null +++ b/test/resources/package_file_01.yaml @@ -0,0 +1,71 @@ +# Top-level comment about the package file +comment: "Main package file for build steps" + +build_steps: + build_step_1: + comment: "First build step, e.g., system dependencies" + phases: + phase_1: + comment: "APT packages for base system" + apt: + comment: "Comment for APT section" + packages: + - name: "curl" + version: "7.68.0" + comment: "For downloading" + - name: "git" + version: "2.25.1" + comment: "VCS support" + pip: + comment: "Python packages" + packages: + - name: "requests" + version: "2.31.0" + comment: "HTTP package" + - name: "click" + version: "8.1.7" + comment: "CLI utilities" + r: + comment: "Optional R packages" + packages: + - name: "ggplot2" + version: "3.4.0" + comment: "Plotting library" + conda: + comment: "Optional conda packages" + channels: + - "defaults" + - "conda-forge" + packages: + - name: "numpy" + version: "1.26.0" + comment: "NumPy for Python" + - name: "pandas" + version: "2.1.2" + comment: "Data manipulation" + phase_2: + comment: "Dev tools for CI/CD" + apt: + comment: "Additional apt packages" + packages: + - name: "python3" + version: "3.8.10" + comment: "System python" + # You can leave out pip/r/conda if not needed in this phase + + build_step_2: + comment: "Second build step, e.g., App build" + phases: + phase_install: + comment: "Install app dependencies" + pip: + packages: + - name: "flask" + version: "3.0.0" + comment: "Web framework" + - name: "sqlalchemy" + version: "2.0.22" + comment: "ORM" + # APT/R/Conda can be omitted if not needed here + +# You can add further build_steps and phases as needed. diff --git a/test/unit/pkg_mgmt/__init__.py b/test/unit/pkg_mgmt/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit/pkg_mgmt/test_apt_install.py b/test/unit/pkg_mgmt/test_apt_install.py new file mode 100644 index 0000000..583abc9 --- /dev/null +++ b/test/unit/pkg_mgmt/test_apt_install.py @@ -0,0 +1,107 @@ +from unittest.mock import ( + MagicMock, + Mock, + call, +) + +import pytest + +from exasol.exaslpm.model.package_file_config import ( + AptPackages, + Package, +) +from exasol.exaslpm.pkg_mgmt.cmd_executor import ( + CommandExecutor, + CommandResult, + StdLogger, +) +from exasol.exaslpm.pkg_mgmt.install_apt import * + +order_of_exec = ["update", "install", "clean", "remove", "locale", "ldconfig"] +call_count = 0 + + +def mock_execute(_, cmd_strs): + global call_count + cmd_str = " ".join(cmd_strs) + assert order_of_exec[call_count] in cmd_str + call_count += 1 + return CommandResult( + StdLogger(), fn_ret_code=lambda: 0, stdout=iter([]), stderr=iter([]) + ) + + +def test_install_via_apt_empty_packages(monkeypatch): + logger = StdLogger() + monkeypatch.setattr(CommandExecutor, "execute", mock_execute) + aptPackages = AptPackages(packages=[]) + install_via_apt(aptPackages, CommandExecutor(logger), logger) + assert "empty list" in logger.get_last_msg() + + +def test_install_via_apt_01(): + mock_executor = MagicMock(spec=CommandExecutor) + pkgs = [ + Package(name="curl", version="7.68.0"), + Package(name="requests", version="2.25.1"), + ] + aptPackages = AptPackages(packages=pkgs) + install_via_apt(aptPackages, mock_executor, StdLogger()) + assert mock_executor.mock_calls == [ + call.execute(["apt-get", "-y", "update"]), + call.execute().print_results(), + call.execute( + [ + "apt-get", + "install", + "-V", + "-y", + "--no-install-recommends", + "curl=7.68.0", + "requests=2.25.1", + ] + ), + call.execute().print_results(), + call.execute(["apt-get", "-y", "clean"]), + call.execute().print_results(), + call.execute(["apt-get", "-y", "autoremove"]), + call.execute().print_results(), + call.execute(["locale-gen", "&&", "update-locale", "LANG=en_US.UTF8"]), + call.execute().print_results(), + call.execute(["ldconfig"]), + call.execute().print_results(), + ] + + +# For Sonar Cube Code Coverage - ToDo: Check once if it complains +def test_prepare_update_command(): + cmd_strs = prepare_update_command() + cmd_str = " ".join(cmd_strs) + assert "apt-get" in cmd_str + assert "update" in cmd_str + + +def test_prepare_clean_cmd(): + cmd_strs = prepare_clean_cmd() + cmd_str = " ".join(cmd_strs) + assert "apt-get" in cmd_str + assert "clean" in cmd_str + + +def test_prepare_autoremove_cmd(): + cmd_strs = prepare_autoremove_cmd() + cmd_str = " ".join(cmd_strs) + assert "apt-get" in cmd_str + assert "remove" in cmd_str + + +def test_prepare_locale_cmd(): + cmd_strs = prepare_locale_cmd() + cmd_str = " ".join(cmd_strs) + assert "locale" in cmd_str + + +def test_prepare_ldconfig_cmd(): + cmd_strs = prepare_ldconfig_cmd() + cmd_str = " ".join(cmd_strs) + assert "ldconfig" in cmd_str diff --git a/test/unit/pkg_mgmt/test_cmd_executor.py b/test/unit/pkg_mgmt/test_cmd_executor.py new file mode 100644 index 0000000..34a5735 --- /dev/null +++ b/test/unit/pkg_mgmt/test_cmd_executor.py @@ -0,0 +1,68 @@ +import subprocess +from unittest.mock import ( + MagicMock, + Mock, + call, +) + +import pytest + +from exasol.exaslpm.pkg_mgmt.cmd_executor import ( + CommandExecutor, + CommandResult, + StdLogger, +) + + +@pytest.fixture +def mock_command_result(): + logger = StdLogger() + fn_ret_code = lambda: 10 + stdout_iter = iter(["stdout line 1", "stdout line 2"]) + stderr_iter = iter(["stderr line 1", "stderr line 2", "stderr line 3"]) + return CommandResult( + logger, fn_ret_code=fn_ret_code, stdout=stdout_iter, stderr=stderr_iter + ) + + +def test_command_executor(monkeypatch): + mock_popen = MagicMock() + monkeypatch.setattr(subprocess, "Popen", mock_popen) + + executor = CommandExecutor(StdLogger()) + result = executor.execute(["cmd1", "cmd2"]) + ret_code = result.return_code() + assert mock_popen.mock_calls == [ + call(["cmd1", "cmd2"], stdout=-1, stderr=-1, text=True), + call().stdout.__iter__(), + call().stderr.__iter__(), + call().wait(), + ] + assert ret_code == mock_popen.return_value.wait.return_value + + +def test_command_results(monkeypatch, mock_command_result): + call_counts = {"stdout": 0, "stderr": 0} + + def mock_execute(_, cmd_strs): + return mock_command_result + + def stdout_results(result_str: str): + call_counts["stdout"] += 1 + expected = "stdout line " + str(call_counts["stdout"]) + assert expected == result_str + + def stderr_results(result_str: str): + call_counts["stderr"] += 1 + expected = "stderr line " + str(call_counts["stderr"]) + assert expected == result_str + + monkeypatch.setattr(CommandExecutor, "execute", mock_execute) + executor = CommandExecutor(StdLogger()) + result = executor.execute(["cmd1", "cmd2"]) + assert result.return_code() == 10 + return_code = result.consume_results(stdout_results, stderr_results) + assert call_counts["stdout"] == 2 + assert call_counts["stderr"] == 3 + assert return_code == 10 + result.print_results()