Build morpheus_llm without building morpheus_core

Signed-off-by: Anuradha Karuppiah <anuradhak@nvidia.com>

cmake/dependencies.cmake

@@ -80,4 +80,8 @@ if(MORPHEUS_BUILD_MORPHEUS_CORE)
  include(dependencies_core)
 endif()
 
+if(MORPHEUS_BUILD_MORPHEUS_LLM)
+ include(dependencies_llm)
+endif()
+
 list(POP_BACK CMAKE_MESSAGE_CONTEXT)

cmake/dependencies_llm.cmake

@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+list(APPEND CMAKE_MESSAGE_CONTEXT "dep_llm")
+
+# pybind11
+# =========
+morpheus_utils_configure_pybind11()
+
+# RD-Kafka
+# =====
+morpheus_utils_configure_rdkafka()
+
+# MRC (Should come after all third party but before NVIDIA repos)
+# =====
+morpheus_utils_configure_mrc()
+
+# CuDF
+morpheus_utils_configure_cudf()
+
+# Triton-client
+# =====
+morpheus_utils_configure_tritonclient()
+
+list(POP_BACK CMAKE_MESSAGE_CONTEXT)

python/morpheus_llm/CMakeLists.txt

@@ -16,7 +16,6 @@
 list(APPEND CMAKE_MESSAGE_CONTEXT "morpheus_llm")
 
 # Save the root of the python for relative paths
-# Where is this used, fixme
 set(MORPHEUS_LLM_PY_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/..)
 
 
@@ -25,16 +24,15 @@ set(MORPHEUS_LLM_PY_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/..)
 # #################################
 morpheus_utils_python_configure()
 
-# Include the main morpheus code
-# fixme: add another subdirectory src and change SOURCE_DIRECTORY to ${CMAKE_CURRENT_SOURCE_DIR}/src
+# Include the main morpheus_llm code
 morpheus_utils_create_python_package(morpheus_llm
  PROJECT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
  SOURCE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/morpheus_llm"
 )
 
 add_subdirectory(morpheus_llm)
 
-# Complete the python package - fixme do these need to be redefined
+# Complete the python package
 if(MORPHEUS_PYTHON_INPLACE_BUILD)
  list(APPEND extra_args "IS_INPLACE")
 endif()

python/morpheus_llm/morpheus_llm/_lib/cmake/libmorpheus_llm.cmake

@@ -27,42 +27,58 @@ add_library(morpheus_llm
 
 add_library(${PROJECT_NAME}::morpheus_llm ALIAS morpheus_llm)
 
-# fixme check if all these are needed
-target_link_libraries(morpheus_llm
- PRIVATE
- matx::matx
- $<$<CONFIG:Debug>:ZLIB::ZLIB>
- PUBLIC
- $<TARGET_NAME_IF_EXISTS:conda_env>
- cudf::cudf
- CUDA::nvtx3
- mrc::pymrc
- ${PROJECT_NAME}::morpheus
-
-)
-
-# Add the include directories of the cudf_helpers_project since we dont want to link directly to it
-get_property(cudf_helpers_target GLOBAL PROPERTY cudf_helpers_target_property)
-get_target_property(cudf_helpers_include ${cudf_helpers_target} INTERFACE_INCLUDE_DIRECTORIES)
+# morpheus_llm can be built two ways -
+# 1. For development purposes (eg. scripts/compile.sh) all the functional blocks are built.
+# This includes morpheus (core), morpheus_llm, morpheus_dfp etc. In this case we
+# set dependencies on build targets across components.
+# 2. For conda packaging purposes morpheus_llm is built on its own. In this case
+# the dependencies (including morpheus-core) are loaded from the conda enviroment.
+if (MORPHEUS_BUILD_MORPHEUS_CORE)
+
+ target_link_libraries(morpheus_llm
+ PRIVATE
+ $<$<CONFIG:Debug>:ZLIB::ZLIB>
+ PUBLIC
+ $<TARGET_NAME_IF_EXISTS:conda_env>
+ cudf::cudf
+ mrc::pymrc
+ ${PROJECT_NAME}::morpheus
+ )
+
+ # Add the include directories of the cudf_helpers_project since we dont want to link directly to it
+ get_property(cudf_helpers_target GLOBAL PROPERTY cudf_helpers_target_property)
+ get_target_property(cudf_helpers_include ${cudf_helpers_target} INTERFACE_INCLUDE_DIRECTORIES)
+
+ target_include_directories(morpheus
+ PRIVATE
+ ${cudf_helpers_include}
+ )
+
+ # Also add a dependency to the target so that the headers are generated before the target is built
+ add_dependencies(morpheus_llm ${cudf_helpers_target})
+
+ # Add a dependency on the morpheus cpython libraries
+ get_property(py_morpheus_target GLOBAL PROPERTY py_morpheus_target_property)
+ add_dependencies(morpheus_llm ${py_morpheus_target})
+
+else()
+
+ rapids_find_package(morpheus REQUIRED)
+
+ target_link_libraries(morpheus_llm
+ PRIVATE
+ $<$<CONFIG:Debug>:ZLIB::ZLIB>
+ PUBLIC
+ $<TARGET_NAME_IF_EXISTS:conda_env>
+ cudf::cudf
+ mrc::pymrc
+ morpheus::morpheus
+ )
 
-target_include_directories(morpheus
- PRIVATE
- ${cudf_helpers_include}
-)
-
-# Also add a dependency to the target so that the headers are generated before the target is built
-add_dependencies(morpheus_llm ${cudf_helpers_target})
-message("add_dependencies morpheus_llm ${cudf_helpers_target}")
-
-# Add a dependency on the morpheus cpython libraries
-get_property(py_morpheus_target GLOBAL PROPERTY py_morpheus_target_property)
-add_dependencies(morpheus_llm ${py_morpheus_target})
-message("add_dependencies morpheus_llm ${py_morpheus_target}")
+endif()
 
-# fixme: find another way to include morpheus headers
 target_include_directories(morpheus_llm
  PUBLIC
- $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src/morpheus/_lib/include>
  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
  $<INSTALL_INTERFACE:include>
 )

scripts/compile.sh

@@ -33,6 +33,7 @@ cmake -S . -B ${BUILD_DIR} -GNinja \
  -DMORPHEUS_USE_CCACHE=ON \
  -DMORPHEUS_USE_CONDA=${MORPHEUS_USE_CONDA:-ON} \
  -DMORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-OFF} \
+ -DMORPHEUS_BUILD_MORPHEUS_CORE=${MORPHEUS_BUILD_MORPHEUS_CORE:-ON} \
  -DMORPHEUS_BUILD_MORPHEUS_LLM=${MORPHEUS_BUILD_MORPHEUS_LLM:-ON} \
  -DMORPHEUS_BUILD_MORPHEUS_DFP=${MORPHEUS_BUILD_MORPHEUS_DFP:-ON} \
  ${INSTALL_PREFIX:+-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}} \