Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: engine management #1546

Merged
merged 16 commits into from
Nov 5, 2024
Merged

feat: engine management #1546

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
59d4a6f
feat: engine management
namchuai Oct 24, 2024
c6bce45
add update engine
namchuai Oct 30, 2024
35c7b6c
store default engine to cortex.rc
namchuai Oct 30, 2024
94060ce
finalizing
namchuai Oct 31, 2024
212749d
temporary remove cli
namchuai Oct 31, 2024
526312a
remove v2 commands
namchuai Oct 31, 2024
6240fc2
Add API references
namchuai Oct 31, 2024
fd961ce
fix ci
namchuai Oct 31, 2024
ae508f3
fix downloading engines with progress bar
namchuai Nov 1, 2024
dc43f6d
add -m flag
namchuai Nov 4, 2024
192a25d
fix ci
namchuai Nov 4, 2024
2848539
clean up
namchuai Nov 4, 2024
7b22b5a
add filter for variant list
namchuai Nov 4, 2024
835a654
update ld_library_path
namchuai Nov 5, 2024
25fc6fd
add dylib for windows
namchuai Nov 5, 2024
4b693f5
add dylib for linux
namchuai Nov 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions engine/cli/commands/server_start_cmd.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,12 +97,8 @@ bool ServerStartCmd::Exec(const std::string& host, int port) {
v += g;
}
CTL_INF("LD_LIBRARY_PATH: " << v);
auto data_path = file_manager_utils::GetEnginesContainerPath();
auto llamacpp_path = data_path / "cortex.llamacpp/";
auto trt_path = data_path / "cortex.tensorrt-llm/";
if (!std::filesystem::exists(llamacpp_path)) {
std::filesystem::create_directory(llamacpp_path);
}
auto llamacpp_path = file_manager_utils::GetCudaToolkitPath(kLlamaRepo);
auto trt_path = file_manager_utils::GetCudaToolkitPath(kTrtLlmRepo);

auto new_v = trt_path.string() + ":" + llamacpp_path.string() + ":" + v;
setenv(name, new_v.c_str(), true);
Expand Down
11 changes: 8 additions & 3 deletions engine/cli/main.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ void SetupLogger(trantor::FileLogger& async_logger, bool verbose) {
std::filesystem::path(config.logFolderPath) /
std::filesystem::path(cortex_utils::logs_folder));
async_logger.setFileName(config.logFolderPath + "/" +
cortex_utils::logs_cli_base_name);
cortex_utils::logs_cli_base_name);
async_logger.setMaxLines(config.maxLogLines); // Keep last 100000 lines
async_logger.startLogging();
trantor::Logger::setOutputFunction(
Expand Down Expand Up @@ -96,15 +96,20 @@ int main(int argc, char* argv[]) {
}
}

{ file_manager_utils::CreateConfigFileIfNotExist(); }
{
auto result = file_manager_utils::CreateConfigFileIfNotExist();
if (result.has_error()) {
CTL_ERR("Error creating config file: " << result.error());
}
}

RemoveBinaryTempFileIfExists();

trantor::FileLogger async_file_logger;
SetupLogger(async_file_logger, verbose);

if (should_install_server) {
InstallServer();
InstallServer();
return 0;
}

Expand Down
12 changes: 5 additions & 7 deletions engine/services/engine_service.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,6 @@ cpp::result<void, std::string> EngineService::InstallEngineAsyncV2(
CTL_INF("InstallEngineAsyncV2: " << ne << ", " << version << ", "
<< variant_name.value_or(""));
auto os = hw_inf_.sys_inf->os;
CTL_INF("os: " << os);
CTL_INF("kMacOs: " << kMacOs);
if (os == kMacOs && (ne == kOnnxRepo || ne == kTrtLlmRepo)) {
return cpp::fail("Engine " + ne + " is not supported on macOS");
}
Expand All @@ -88,9 +86,9 @@ cpp::result<void, std::string> EngineService::InstallEngineAsyncV2(

auto result = DownloadEngineV2(ne, version, variant_name);
if (result.has_error()) {
return result;
return cpp::fail(result.error());
}
auto cuda_res = DownloadCuda(ne, true /*async*/);
auto cuda_res = DownloadCuda(ne, true);
if (cuda_res.has_error()) {
return cpp::fail(cuda_res.error());
}
Expand Down Expand Up @@ -451,12 +449,12 @@ cpp::result<bool, std::string> EngineService::DownloadCuda(

auto cuda_toolkit_url = url_parser::FromUrl(url_obj);

LOG_DEBUG << "Cuda toolkit download url: " << cuda_toolkit_url;
CTL_DBG("Cuda toolkit download url: " << cuda_toolkit_url);
auto cuda_toolkit_local_path =
file_manager_utils::GetContainerFolderPath(
file_manager_utils::DownloadTypeToString(DownloadType::CudaToolkit)) /
cuda_toolkit_file_name;
LOG_DEBUG << "Download to: " << cuda_toolkit_local_path.string();
CTL_DBG("Download to: " << cuda_toolkit_local_path.string());
auto downloadCudaToolkitTask{DownloadTask{
.id = download_id,
.type = DownloadType::CudaToolkit,
Expand All @@ -466,7 +464,7 @@ cpp::result<bool, std::string> EngineService::DownloadCuda(
}};

auto on_finished = [engine](const DownloadTask& finishedTask) {
auto engine_path = file_manager_utils::GetEnginesContainerPath() / engine;
auto engine_path = file_manager_utils::GetCudaToolkitPath(engine);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we rename to cuda_path?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will update in following pr.

archive_utils::ExtractArchive(finishedTask.items[0].localPath.string(),
engine_path.string());

Expand Down
13 changes: 13 additions & 0 deletions engine/utils/file_manager_utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,19 @@ inline std::filesystem::path GetModelsContainerPath() {
return models_container_path;
}

inline std::filesystem::path GetCudaToolkitPath(const std::string& engine) {
auto engine_path = getenv("ENGINE_PATH")
? std::filesystem::path(getenv("ENGINE_PATH"))
: GetCortexDataPath();

auto cuda_path = engine_path / "engines" / engine / "deps";
if (!std::filesystem::exists(cuda_path)) {
std::filesystem::create_directories(cuda_path);
}

return cuda_path;
}

inline std::filesystem::path GetEnginesContainerPath() {
auto cortex_path = getenv("ENGINE_PATH")
? std::filesystem::path(getenv("ENGINE_PATH"))
Expand Down