[Fix Hourly] install UCX from source instead using builtin wheel from nixl

.github/workflows/pre-merge.yaml

@@ -373,7 +373,7 @@ jobs:
             -v /mnt/hf_cache:/workspace/hf_cache \
             hpu-plugin-v1-test-env-pre-merge-${{ github.event.pull_request.head.sha }} \
             /bin/bash -c "
-                pip install nixl==0.5.0 lm-eval[api] &&
+                pip install lm-eval[api] &&
                 cd /workspace/vllm-gaudi/tests/unit_tests &&
                 ./run_accuracy_test.sh
             "

examples/nixl/requirements.txt

@@ -1,4 +1,4 @@
 pytest
-nixl==0.5.0
+nixl
 lm-eval
 lm-eval[api]

install_nixl.py

@@ -0,0 +1,147 @@
+# install_prerequisites.py
+import os
+import subprocess
+import sys
+import argparse
+
+# --- Configuration ---
+ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+UCX_DIR = 'ucx_source'
+UCX_REPO_URL = 'https://github.com/openucx/ucx.git'
+UCX_INSTALL_DIR = os.path.join(ROOT_DIR, 'ucx_install')
+NIXL_DIR = 'nixl_source'
+NIXL_REPO_URL = 'https://github.com/ai-dynamo/nixl.git'
+
+
+# --- Helper Functions ---
+def run_command(command, cwd='.', env=None):
+    """Helper function to run a shell command and check for errors."""
+    print(f"--> Running command: {' '.join(command)} in '{cwd}'")
+    subprocess.check_call(command, cwd=cwd, env=env)
+
+
+def is_pip_package_installed(package_name):
+    """Checks if a package is installed via pip without raising an exception."""
+    result = subprocess.run([sys.executable, '-m', 'pip', 'show', package_name],
+                            stdout=subprocess.DEVNULL,
+                            stderr=subprocess.DEVNULL)
+    return result.returncode == 0
+
+
+def install_system_dependencies():
+    """Installs required system packages using apt-get if run as root."""
+    if os.geteuid() != 0:
+        print("\n---")
+        print("WARNING: Not running as root. Skipping system dependency installation.")
+        print("Please ensure the following packages are installed on your system:")
+        print("  build-essential git cmake ninja-build autotools-dev automake meson libtool libtool-bin")
+        print("---\n")
+        return
+
+    print("--- Running as root. Installing system dependencies... ---")
+    apt_packages = [
+        "build-essential", "git", "cmake", "ninja-build", "autotools-dev", "automake", "meson", "libtool", "libtool-bin"
+    ]
+    run_command(['apt-get', 'update'])
+    run_command(['apt-get', 'install', '-y'] + apt_packages)
+    print("--- System dependencies installed successfully. ---\n")
+
+
+def update_nixl_config():
+    """Programmatically updates the nixl pyproject.toml file."""
+    try:
+        import tomli
+        import tomli_w
+    except ModuleNotFoundError:
+        print("--> Dependency 'tomli' or 'tomli-w' not found. Installing now...")
+        run_command([sys.executable, '-m', 'pip', 'install', 'tomli', 'tomli-w'])
+        import tomli
+        import tomli_w
+
+    file_path = os.path.join(NIXL_DIR, 'pyproject.toml')
+    print(f"--> Updating configuration in {file_path}")
+    try:
+        with open(file_path, 'rb') as f:
+            data = tomli.load(f)
+        args = data.setdefault('tool', {}).setdefault('meson-python', {}).setdefault('args', {})
+        args['setup'] = ['-Dinstall_headers=false', '-Ddisable_gds_backend=true']
+        with open(file_path, 'wb') as f:
+            tomli_w.dump(data, f)
+        print("--> NIXL configuration updated successfully.")
+    except Exception as e:
+        print(f"ERROR: Failed to update nixl config: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def build_and_install_prerequisites(args):
+    """Builds UCX and NIXL from source, with checks to skip if already installed."""
+    install_system_dependencies()
+    print("--- Starting prerequisite build and install process ---")
+    ucx_install_path = os.path.abspath(UCX_INSTALL_DIR)
+
+    # -- Step 1: Build and Install UCX from source --
+    ucx_check_file = os.path.join(ucx_install_path, 'bin', 'ucx_info')
+    if not args.force_reinstall and os.path.exists(ucx_check_file):
+        print("\n--> UCX already found. Skipping build. Use --force-reinstall to override.")
+    else:
+        print("\n[1/2] Configuring and building UCX from source...")
+        if not os.path.exists(UCX_DIR):
+            run_command(['git', 'clone', UCX_REPO_URL, UCX_DIR])
+
+        ucx_source_path = os.path.abspath(UCX_DIR)
+        run_command(['git', 'checkout', 'v1.19.x'], cwd=ucx_source_path)
+        run_command(['./autogen.sh'], cwd=ucx_source_path)
+
+        configure_command = [
+            './configure',
+            f'--prefix={ucx_install_path}',
+            '--enable-shared',
+            '--disable-static',
+            '--disable-doxygen-doc',
+            '--enable-optimizations',
+            '--enable-cma',
+            '--enable-devel-headers',
+            '--with-verbs',
+            '--enable-mt',
+        ]
+        run_command(configure_command, cwd=ucx_source_path)
+        run_command(['make', '-j', str(os.cpu_count() or 1)], cwd=ucx_source_path)
+        run_command(['make', 'install'], cwd=ucx_source_path)
+        print("--- UCX build and install complete ---")
+
+    # -- Step 2: Build and Install NIXL from source --
+    if not args.force_reinstall and is_pip_package_installed('nixl'):
+        print("\n--> NIXL is already installed. Skipping build. Use --force-reinstall to override.")
+    else:
+        print("\n[2/2] Configuring and building NIXL from source...")
+        if not os.path.exists(NIXL_DIR):
+            run_command(['git', 'clone', NIXL_REPO_URL, NIXL_DIR])
+
+        update_nixl_config()
+
+        build_env = os.environ.copy()
+        pkg_config_path = os.path.join(ucx_install_path, 'lib', 'pkgconfig')
+        build_env['PKG_CONFIG_PATH'] = pkg_config_path
+
+        ucx_lib_path = os.path.join(ucx_install_path, 'lib')
+        existing_ld_path = os.environ.get('LD_LIBRARY_PATH', '')
+        build_env['LD_LIBRARY_PATH'] = f"{ucx_lib_path}:{existing_ld_path}".strip(':')
+
+        print(f"--> Using PKG_CONFIG_PATH: {build_env['PKG_CONFIG_PATH']}")
+        print(f"--> Using LD_LIBRARY_PATH: {build_env['LD_LIBRARY_PATH']}")
+
+        nixl_install_command = [sys.executable, '-m', 'pip', 'install', '--no-cache-dir', '.']
+        if args.force_reinstall:
+            nixl_install_command.insert(-1, '--force-reinstall')
+
+        run_command(nixl_install_command, cwd=os.path.abspath(NIXL_DIR), env=build_env)
+        print("--- NIXL installation complete ---")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Build and install UCX and NIXL dependencies.")
+    parser.add_argument('--force-reinstall',
+                        action='store_true',
+                        help='Force rebuild and reinstall of UCX and NIXL even if they are already installed.')
+    args = parser.parse_args()
+    build_and_install_prerequisites(args)

requirements.txt

@@ -7,4 +7,3 @@ setuptools>=77.0.3,<80.0.0
 setuptools-scm>=8
 numba
 transformers>=4.1,<4.56.0
-nixl==0.5.0

tests/unit_tests/run_accuracy_test.sh

@@ -1,5 +1,17 @@
 #!/bin/bash
-set -xe
+set -e
+
+# --- Install Nixl ---
+echo "Installing lm-eval dependency..."
+python ../../install_nixl.py
+echo "Dependency installation complete."
+
+
+# --- Install Dependencies ---
+echo "Installing lm-eval dependency..."
+pip install lm-eval[api]
+echo "Dependency installation complete."
+
 
 # Models to run
 MODELS=(