diff --git a/engine/cli/commands/engine_get_cmd.cc b/engine/cli/commands/engine_get_cmd.cc index 992f7267b..8699c336b 100644 --- a/engine/cli/commands/engine_get_cmd.cc +++ b/engine/cli/commands/engine_get_cmd.cc @@ -35,7 +35,14 @@ void EngineGetCmd::Exec(const std::string& host, int port, }; auto result = curl_utils::SimpleGetJson(url.ToFullPath()); if (result.has_error()) { - CTL_ERR(result.error()); + // TODO: refactor this + Json::Value root; + Json::Reader reader; + if (!reader.parse(result.error(), root)) { + CLI_LOG(result.error()); + return; + } + CLI_LOG(root["message"].asString()); return; } diff --git a/engine/cli/commands/engine_install_cmd.cc b/engine/cli/commands/engine_install_cmd.cc index 2f6129a35..72b962a0b 100644 --- a/engine/cli/commands/engine_install_cmd.cc +++ b/engine/cli/commands/engine_install_cmd.cc @@ -9,6 +9,14 @@ namespace commands { bool EngineInstallCmd::Exec(const std::string& engine, const std::string& version, const std::string& src) { + // Start server if server is not started yet + if (!commands::IsServerAlive(host_, port_)) { + CLI_LOG("Starting server ..."); + commands::ServerStartCmd ssc; + if (!ssc.Exec(host_, port_)) { + return false; + } + } // Handle local install, if fails, fallback to remote install if (!src.empty()) { auto res = engine_service_.UnzipEngine(engine, version, src); @@ -23,20 +31,12 @@ bool EngineInstallCmd::Exec(const std::string& engine, } if (show_menu_) { - // Start server if server is not started yet - if (!commands::IsServerAlive(host_, port_)) { - CLI_LOG("Starting server ..."); - commands::ServerStartCmd ssc; - if (!ssc.Exec(host_, port_)) { - return false; - } - } - DownloadProgress dp; dp.Connect(host_, port_); // engine can be small, so need to start ws first - auto dp_res = std::async(std::launch::deferred, - [&dp, &engine] { return dp.Handle(engine); }); + auto dp_res = std::async(std::launch::deferred, [&dp, &engine] { + return dp.Handle(DownloadType::Engine); + }); CLI_LOG("Validating download items, please wait..") auto versions_url = url_parser::Url{ @@ -118,7 +118,7 @@ bool EngineInstallCmd::Exec(const std::string& engine, bool check_cuda_download = !system_info_utils::GetCudaVersion().empty(); if (check_cuda_download) { - if (!dp.Handle("cuda")) + if (!dp.Handle(DownloadType::CudaToolkit)) return false; } @@ -130,10 +130,8 @@ bool EngineInstallCmd::Exec(const std::string& engine, DownloadProgress dp; dp.Connect(host_, port_); // engine can be small, so need to start ws first - auto dp_res = std::async(std::launch::deferred, [&dp] { - return dp.Handle(DownloadType::Engine); - }); - CLI_LOG("Validating download items, please wait..") + auto dp_res = std::async(std::launch::deferred, + [&dp] { return dp.Handle(DownloadType::Engine); }); auto install_url = url_parser::Url{ .protocol = "http", @@ -146,12 +144,25 @@ bool EngineInstallCmd::Exec(const std::string& engine, }, }; + if (!version.empty()) { + install_url.queries = {{"version", version}}; + } + auto response = curl_utils::SimplePostJson(install_url.ToFullPath()); if (response.has_error()) { - CTL_ERR(response.error()); + // TODO: namh refactor later + Json::Value root; + Json::Reader reader; + if (!reader.parse(response.error(), root)) { + CLI_LOG(response.error()); + return false; + } + CLI_LOG(root["message"].asString()); return false; } + CLI_LOG("Validating download items, please wait..") + if (!dp_res.get()) return false; diff --git a/engine/cli/commands/engine_release_cmd.cc b/engine/cli/commands/engine_release_cmd.cc deleted file mode 100644 index e3dde9fe9..000000000 --- a/engine/cli/commands/engine_release_cmd.cc +++ /dev/null @@ -1,86 +0,0 @@ -#include "engine_release_cmd.h" -#include -#include "commands/server_start_cmd.h" -#include "utils/cli_selection_utils.h" -#include "utils/curl_utils.h" -#include "utils/github_release_utils.h" -#include "utils/logging_utils.h" -#include "utils/url_parser.h" - -namespace commands { - -cpp::result EngineReleaseCmd::Exec( - const std::string& host, int port, const std::string& engine_name) { - - // Start server if server is not started yet - if (!commands::IsServerAlive(host, port)) { - CLI_LOG("Starting server ..."); - commands::ServerStartCmd ssc; - if (!ssc.Exec(host, port)) { - return cpp::fail("Failed to start server"); - } - } - - auto url_obj = url_parser::Url{ - .protocol = "http", - .host = host + ":" + std::to_string(port), - .pathParams = {"v1", "engines", engine_name, "releases"}, - }; - - auto res = curl_utils::SimpleGetJson(url_obj.ToFullPath()); - if (res.has_error()) { - CLI_LOG("url: " + url_obj.ToFullPath()); - return cpp::fail("Failed to get engine release: " + engine_name + - ", error: " + res.error()); - } - if (res.value().size() == 0) { - return cpp::fail("No release found for engine: " + engine_name); - } - - std::vector selections; - for (const auto& release : res.value()) { - selections.push_back(release["tag_name"].asString()); - } - - auto selection = - cli_selection_utils::PrintSelection(selections, "Available versions:"); - if (!selection.has_value()) { - return cpp::fail("Invalid selection!"); // TODO: return latest version - } - - CLI_LOG("Selected " + engine_name + " version " + selection.value()); - Json::Value selected_release; - for (const auto& release : res.value()) { - if (release["tag_name"].asString() == selection.value()) { - selected_release = release; - break; - } - } - - std::vector variant_selections; - for (const auto& variant : selected_release["assets"]) { - variant_selections.push_back(variant["name"].asString()); - } - - auto variant_selection = cli_selection_utils::PrintSelection( - variant_selections, "Available variant:"); - if (!variant_selection.has_value()) { - return cpp::fail("Invalid variant selection!"); - } - - CLI_LOG("Selected " + variant_selection.value()); - github_release_utils::GitHubAsset selected_asset; - for (const auto& asset : selected_release["assets"]) { - if (asset["name"] == variant_selection) { - auto version = string_utils::RemoveSubstring(selection.value(), "v"); - selected_asset = - github_release_utils::GitHubAsset::FromJson(asset, version); - break; - } - } - - // proceed to download - - return {}; -} -}; // namespace commands diff --git a/engine/cli/commands/engine_release_cmd.h b/engine/cli/commands/engine_release_cmd.h deleted file mode 100644 index a292d81cb..000000000 --- a/engine/cli/commands/engine_release_cmd.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include -#include "utils/result.hpp" - -namespace commands { -class EngineReleaseCmd { - public: - cpp::result Exec(const std::string& host, int port, - const std::string& engine_name); -}; - -} // namespace commands diff --git a/engine/cli/commands/engine_update_cmd.cc b/engine/cli/commands/engine_update_cmd.cc index 71cb2dfeb..b9e3acf1b 100644 --- a/engine/cli/commands/engine_update_cmd.cc +++ b/engine/cli/commands/engine_update_cmd.cc @@ -23,8 +23,9 @@ bool EngineUpdateCmd::Exec(const std::string& host, int port, DownloadProgress dp; dp.Connect(host, port); // engine can be small, so need to start ws first - auto dp_res = std::async(std::launch::deferred, - [&dp, &engine] { return dp.Handle(engine); }); + auto dp_res = std::async(std::launch::deferred, [&dp, &engine] { + return dp.Handle(DownloadType::Engine); + }); CLI_LOG("Validating download items, please wait..") auto update_url = url_parser::Url{ @@ -43,7 +44,7 @@ bool EngineUpdateCmd::Exec(const std::string& host, int port, bool check_cuda_download = !system_info_utils::GetCudaVersion().empty(); if (check_cuda_download) { - if (!dp.Handle("cuda")) + if (!dp.Handle(DownloadType::CudaToolkit)) return false; } diff --git a/engine/controllers/engines.cc b/engine/controllers/engines.cc index 9673ef29a..63f86ed38 100644 --- a/engine/controllers/engines.cc +++ b/engine/controllers/engines.cc @@ -29,8 +29,11 @@ void Engines::ListEngine( for (const auto& engine : supported_engines) { auto installed_engines = engine_service_->GetInstalledEngineVariants(engine); + if (installed_engines.has_error()) { + continue; + } Json::Value variants(Json::arrayValue); - for (const auto& variant : installed_engines) { + for (const auto& variant : installed_engines.value()) { variants.append(variant.ToJson()); } ret[engine] = variants; @@ -157,8 +160,16 @@ void Engines::GetInstalledEngineVariants( std::function&& callback, const std::string& engine) const { auto result = engine_service_->GetInstalledEngineVariants(engine); + if (result.has_error()) { + Json::Value res; + res["message"] = result.error(); + auto resp = cortex_utils::CreateCortexHttpJsonResponse(res); + resp->setStatusCode(k400BadRequest); + callback(resp); + return; + } Json::Value releases(Json::arrayValue); - for (const auto& variant : result) { + for (const auto& variant : result.value()) { releases.append(variant.ToJson()); } auto resp = cortex_utils::CreateCortexHttpJsonResponse(releases); diff --git a/engine/e2e-test/test_api_model_delete.py b/engine/e2e-test/test_api_model_delete.py index f45768c66..7415a3d5a 100644 --- a/engine/e2e-test/test_api_model_delete.py +++ b/engine/e2e-test/test_api_model_delete.py @@ -1,6 +1,5 @@ import pytest import requests -from test_runner import popen, run from test_runner import start_server, stop_server diff --git a/engine/e2e-test/test_cli_engine_get.py b/engine/e2e-test/test_cli_engine_get.py index d783c3421..d1cef3a8b 100644 --- a/engine/e2e-test/test_cli_engine_get.py +++ b/engine/e2e-test/test_cli_engine_get.py @@ -1,11 +1,11 @@ import platform import pytest -from test_runner import run -from test_runner import start_server, stop_server +from test_runner import run, start_server, stop_server + class TestCliEngineGet: - + @pytest.fixture(autouse=True) def setup_and_teardown(self): # Setup @@ -20,9 +20,7 @@ def setup_and_teardown(self): @pytest.mark.skipif(platform.system() != "Windows", reason="Windows-specific test") def test_engines_get_tensorrt_llm_should_not_be_incompatible(self): - exit_code, output, error = run( - "Get engine", ["engines", "get", "tensorrt-llm"] - ) + exit_code, output, error = run("Get engine", ["engines", "get", "tensorrt-llm"]) assert exit_code == 0, f"Get engine failed with error: {error}" assert ( "Incompatible" not in output @@ -37,9 +35,7 @@ def test_engines_get_onnx_should_not_be_incompatible(self): ), "onnxruntime should be Ready or Not Installed on Windows" def test_engines_get_llamacpp_should_not_be_incompatible(self): - exit_code, output, error = run( - "Get engine", ["engines", "get", "llama-cpp"] - ) + exit_code, output, error = run("Get engine", ["engines", "get", "llama-cpp"]) assert exit_code == 0, f"Get engine failed with error: {error}" assert ( "Incompatible" not in output @@ -47,19 +43,19 @@ def test_engines_get_llamacpp_should_not_be_incompatible(self): @pytest.mark.skipif(platform.system() != "Darwin", reason="macOS-specific test") def test_engines_get_tensorrt_llm_should_be_incompatible_on_macos(self): - exit_code, output, error = run( - "Get engine", ["engines", "get", "tensorrt-llm"] - ) + exit_code, output, error = run("Get engine", ["engines", "get", "tensorrt-llm"]) assert exit_code == 0, f"Get engine failed with error: {error}" assert ( - "Incompatible" in output + "is not supported on" in output ), "tensorrt-llm should be Incompatible on MacOS" @pytest.mark.skipif(platform.system() != "Darwin", reason="macOS-specific test") def test_engines_get_onnx_should_be_incompatible_on_macos(self): exit_code, output, error = run("Get engine", ["engines", "get", "onnxruntime"]) assert exit_code == 0, f"Get engine failed with error: {error}" - assert "Incompatible" in output, "onnxruntime should be Incompatible on MacOS" + assert ( + "is not supported on" in output + ), "onnxruntime should be Incompatible on MacOS" @pytest.mark.skipif(platform.system() != "Linux", reason="Linux-specific test") def test_engines_get_onnx_should_be_incompatible_on_linux(self): diff --git a/engine/e2e-test/test_cli_engine_install.py b/engine/e2e-test/test_cli_engine_install.py index b3d4f696d..6c8c4932b 100644 --- a/engine/e2e-test/test_cli_engine_install.py +++ b/engine/e2e-test/test_cli_engine_install.py @@ -1,10 +1,9 @@ -import os import platform import tempfile -from pathlib import Path import pytest -from test_runner import run +import requests +from test_runner import run, start_server, stop_server class TestCliEngineInstall: @@ -26,10 +25,8 @@ def test_engines_install_llamacpp_should_be_successfully(self): timeout=None, capture=False, ) - root = Path.home() - assert os.path.exists( - root / "cortexcpp" / "engines" / "cortex.llamacpp" / "version.txt" - ) + response = requests.get("http://127.0.0.1:3928/v1/engines/llama-cpp") + assert len(response.json()) > 0 assert exit_code == 0, f"Install engine failed with error: {error}" @pytest.mark.skipif(platform.system() != "Darwin", reason="macOS-specific test") @@ -37,7 +34,7 @@ def test_engines_install_onnx_on_macos_should_be_failed(self): exit_code, output, error = run( "Install Engine", ["engines", "install", "onnxruntime"] ) - assert "No variant found" in output, "Should display error message" + assert "is not supported on" in output, "Should display error message" assert exit_code == 0, f"Install engine failed with error: {error}" @pytest.mark.skipif(platform.system() != "Darwin", reason="macOS-specific test") @@ -45,20 +42,28 @@ def test_engines_install_onnx_on_tensorrt_should_be_failed(self): exit_code, output, error = run( "Install Engine", ["engines", "install", "tensorrt-llm"] ) - assert "No variant found" in output, "Should display error message" + assert "is not supported on" in output, "Should display error message" assert exit_code == 0, f"Install engine failed with error: {error}" def test_engines_install_pre_release_llamacpp(self): + engine_version = "v0.1.29" exit_code, output, error = run( "Install Engine", - ["engines", "install", "llama-cpp", "-v", "v0.1.29"], + ["engines", "install", "llama-cpp", "-v", engine_version], timeout=None, capture=False, ) - root = Path.home() - assert os.path.exists( - root / "cortexcpp" / "engines" / "cortex.llamacpp" / "version.txt" - ) + response = requests.get("http://127.0.0.1:3928/v1/engines/llama-cpp") + assert len(response.json()) > 0 + is_engine_version_exist = False + for item in response.json(): + # Check if 'version' key exists and matches target + if "version" in item and item["version"] == engine_version: + is_engine_version_exist = True + break + + # loop through all the installed response, expect we find + assert is_engine_version_exist, f"Engine version {engine_version} is not found" assert exit_code == 0, f"Install engine failed with error: {error}" def test_engines_should_fallback_to_download_llamacpp_engine_if_not_exists(self): @@ -67,7 +72,9 @@ def test_engines_should_fallback_to_download_llamacpp_engine_if_not_exists(self) ["engines", "install", "llama-cpp", "-s", tempfile.gettempdir()], timeout=None, ) - assert "Start downloading" in output, "Should display downloading message" + # response = requests.get("http://127.0.0.1:3928/v1/engines/llama-cpp") + # assert len(response.json()) > 0 + assert "downloaded successfully" in output assert exit_code == 0, f"Install engine failed with error: {error}" def test_engines_should_not_perform_with_dummy_path(self): diff --git a/engine/e2e-test/test_cli_engine_list.py b/engine/e2e-test/test_cli_engine_list.py index 210e439cf..99b65d860 100644 --- a/engine/e2e-test/test_cli_engine_list.py +++ b/engine/e2e-test/test_cli_engine_list.py @@ -28,11 +28,9 @@ def test_engines_list_run_successfully_on_windows(self): def test_engines_list_run_successfully_on_macos(self): exit_code, output, error = run("List engines", ["engines", "list"]) assert exit_code == 0, f"List engines failed with error: {error}" - assert "llama-cpp" in output @pytest.mark.skipif(platform.system() != "Linux", reason="Linux-specific test") def test_engines_list_run_successfully_on_linux(self): exit_code, output, error = run("List engines", ["engines", "list"]) assert exit_code == 0, f"List engines failed with error: {error}" assert "llama-cpp" in output - diff --git a/engine/e2e-test/test_cortex_update.py b/engine/e2e-test/test_cortex_update.py index 2d7d652ec..8f6f8d7f8 100644 --- a/engine/e2e-test/test_cortex_update.py +++ b/engine/e2e-test/test_cortex_update.py @@ -1,7 +1,8 @@ +import os +import tempfile + import pytest from test_runner import run -import tempfile -import os class TestCortexUpdate: @@ -12,4 +13,4 @@ def test_cortex_update(self): exit_code, output, error = run("Update cortex", ["update"]) assert exit_code == 0, "Something went wrong" assert "Updated cortex sucessfully" in output - assert os.path.exists(os.path.join(tempfile.gettempdir()), 'cortex') == False + assert os.path.exists(os.path.join(tempfile.gettempdir()), "cortex") == False diff --git a/engine/services/engine_service.cc b/engine/services/engine_service.cc index cc94bd943..ec5417b7e 100644 --- a/engine/services/engine_service.cc +++ b/engine/services/engine_service.cc @@ -75,6 +75,16 @@ cpp::result EngineService::InstallEngineAsyncV2( auto ne = NormalizeEngine(engine); CTL_INF("InstallEngineAsyncV2: " << ne << ", " << version << ", " << variant_name.value_or("")); + auto os = hw_inf_.sys_inf->os; + CTL_INF("os: " << os); + CTL_INF("kMacOs: " << kMacOs); + if (os == kMacOs && (ne == kOnnxRepo || ne == kTrtLlmRepo)) { + return cpp::fail("Engine " + ne + " is not supported on macOS"); + } + + if (os == kLinuxOs && ne == kOnnxRepo) { + return cpp::fail("Engine " + ne + " is not supported on Linux"); + } auto result = DownloadEngineV2(ne, version, variant_name); if (result.has_error()) { @@ -570,14 +580,18 @@ cpp::result EngineService::IsEngineVariantReady( auto ne = NormalizeEngine(engine); auto normalized_version = string_utils::RemoveSubstring(version, "v"); auto installed_engines = GetInstalledEngineVariants(ne); + if (installed_engines.has_error()) { + return cpp::fail(installed_engines.error()); + } CLI_LOG("IsEngineVariantReady: " << ne << ", " << normalized_version << ", " << variant); - for (const auto& installed_engine : installed_engines) { + for (const auto& installed_engine : installed_engines.value()) { CLI_LOG("Installed: name: " + installed_engine.name + ", version: " + installed_engine.version); if (installed_engine.name == variant && - installed_engine.version == normalized_version) { + installed_engine.version == normalized_version || + installed_engine.version == "v" + normalized_version) { return true; } } @@ -608,9 +622,18 @@ EngineService::GetDefaultEngineVariant(const std::string& engine) { }; } -std::vector EngineService::GetInstalledEngineVariants( - const std::string& engine) const { +cpp::result, std::string> +EngineService::GetInstalledEngineVariants(const std::string& engine) const { auto ne = NormalizeEngine(engine); + auto os = hw_inf_.sys_inf->os; + if (os == kMacOs && (ne == kOnnxRepo || ne == kTrtLlmRepo)) { + return cpp::fail("Engine " + engine + " is not supported on macOS"); + } + + if (os == kLinuxOs && ne == kOnnxRepo) { + return cpp::fail("Engine " + engine + " is not supported on Linux"); + } + auto engines_variants_dir = file_manager_utils::GetEnginesContainerPath() / ne; @@ -635,7 +658,7 @@ std::vector EngineService::GetInstalledEngineVariants( auto node = YAML::LoadFile(version_txt_path.string()); auto ev = EngineVariantResponse{ .name = node["name"].as(), - .version = node["version"].as(), + .version = "v" + node["version"].as(), .engine = engine, }; variants.push_back(ev); @@ -816,7 +839,6 @@ EngineService::GetLatestEngineVersion(const std::string& engine) const { cpp::result EngineService::IsEngineReady( const std::string& engine) const { auto ne = NormalizeEngine(engine); - auto installed_variants = GetInstalledEngineVariants(engine); auto os = hw_inf_.sys_inf->os; if (os == kMacOs && (ne == kOnnxRepo || ne == kTrtLlmRepo)) { @@ -826,8 +848,12 @@ cpp::result EngineService::IsEngineReady( if (os == kLinuxOs && ne == kOnnxRepo) { return cpp::fail("Engine " + engine + " is not supported on Linux"); } + auto installed_variants = GetInstalledEngineVariants(engine); + if (installed_variants.has_error()) { + return cpp::fail(installed_variants.error()); + } - return installed_variants.size() > 0; + return installed_variants->size() > 0; } cpp::result EngineService::UpdateEngine( @@ -856,7 +882,7 @@ cpp::result EngineService::UpdateEngine( auto installed_variants = GetInstalledEngineVariants(ne); bool latest_version_installed = false; - for (const auto& v : installed_variants) { + for (const auto& v : installed_variants.value()) { CTL_INF("Installed version: " + v.version); CTL_INF(json_helper::DumpJsonString(v.ToJson())); if (default_variant->variant == v.name && diff --git a/engine/services/engine_service.h b/engine/services/engine_service.h index d10f29678..5b99e31dd 100644 --- a/engine/services/engine_service.h +++ b/engine/services/engine_service.h @@ -128,8 +128,8 @@ class EngineService { cpp::result GetDefaultEngineVariant( const std::string& engine); - std::vector GetInstalledEngineVariants( - const std::string& engine) const; + cpp::result, std::string> + GetInstalledEngineVariants(const std::string& engine) const; bool IsEngineLoaded(const std::string& engine) const; diff --git a/engine/utils/curl_utils.h b/engine/utils/curl_utils.h index 01048f82b..88b05828a 100644 --- a/engine/utils/curl_utils.h +++ b/engine/utils/curl_utils.h @@ -65,8 +65,7 @@ inline cpp::result SimpleGet(const std::string& url) { if (http_code >= 400) { CTL_ERR("HTTP request failed with status code: " + std::to_string(http_code)); - return cpp::fail("API request failed: " + - static_cast(curl_easy_strerror(res))); + return cpp::fail(readBuffer); } return readBuffer; @@ -117,8 +116,7 @@ inline cpp::result SimplePost( if (http_code >= 400) { CTL_ERR("HTTP request failed with status code: " + std::to_string(http_code)); - return cpp::fail("API request failed: " + - static_cast(curl_easy_strerror(res))); + return cpp::fail(readBuffer); } return readBuffer;