From fae0bcdb559dafa9382b10f9ea824d500f655b34 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 22 Feb 2024 18:11:29 +0000
Subject: [PATCH 01/47] Remove install_dependencies from buld.sh call

---
 .github/workflows/ci-cpu-cpp.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index ec0a9c88d8..0f920d389d 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -31,4 +31,7 @@ jobs:
           python ts_scripts/install_dependencies.py --environment=dev --cpp
       - name: Build
         run: |
-          cd cpp && ./build.sh --install-dependencies
+          cd cpp && ./build.sh
+      - name: Install TorchServe
+        run: |
+          cd .. && python ts_scripts/install_from_src.py

From 801db4a52df7c02e8aa0665f096aad5a67f8ac6f Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 22 Feb 2024 18:34:09 +0000
Subject: [PATCH 02/47] remove sudo from folly build

---
 cpp/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/build.sh b/cpp/build.sh
index 57fad8c8e8..bec3e04233 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -42,7 +42,7 @@ function install_folly() {
     fi
     $SUDO ./build/fbcode_builder/getdeps.py install-system-deps --recursive
 
-    $SUDO ./build/fbcode_builder/getdeps.py build \
+    ./build/fbcode_builder/getdeps.py build \
     --allow-system-packages \
     --scratch-path $FOLLY_BUILD_DIR \
     --extra-cmake-defines='{"CMAKE_CXX_FLAGS": "-fPIC -D_GLIBCXX_USE_CXX11_ABI=1"}'

From f1ff55ba051cf5fc5d7567b9336874caa64d68ff Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 22 Feb 2024 19:34:04 +0000
Subject: [PATCH 03/47] Align cpp wf file with cpu wf file

---
 .github/workflows/ci-cpu-cpp.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 0f920d389d..12dc215e55 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -18,10 +18,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macOS-latest]
+        os: [ubuntu-20.04, macOS-latest]
     steps:
       - name: Checkout TorchServe
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Install libtorch - macOS
         if: matrix.os == 'macOS-latest'
         run: |

From c6996c9163e80745e6a07f1b91d929f29e56d568 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 22 Feb 2024 21:19:04 +0000
Subject: [PATCH 04/47] Make yaml statically linked dependency

---
 cpp/CMakeLists.txt | 19 +++++++++++++++----
 cpp/build.sh       | 41 -----------------------------------------
 2 files changed, 15 insertions(+), 45 deletions(-)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 01bf8b9ce8..2a95750bd1 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -18,10 +18,6 @@ if(CLANG_FORMAT_EXE)
     ${PROJECT_SOURCE_DIR}/test/*.hh
   )
 
-  add_custom_target(format
-    COMMAND
-      ${CLANG_FORMAT_EXE} -i -style=google ${ALL_CXX_SOURCE_FILES}
-  )
 endif()
 
 
@@ -31,6 +27,21 @@ find_package(fmt REQUIRED)
 find_package(gflags REQUIRED)
 find_package(Torch REQUIRED)
 
+include(FetchContent)
+
+FetchContent_Declare(
+  yaml-cpp
+  GIT_REPOSITORY https://github.com/jbeder/yaml-cpp.git
+  GIT_TAG 0.8.0 # Can be a tag (yaml-cpp-x.x.x), a commit hash, or a branch name (master)
+)
+FetchContent_GetProperties(yaml-cpp)
+
+if(NOT yaml-cpp_POPULATED)
+  message(STATUS "Fetching yaml-cpp...")
+  FetchContent_Populate(yaml-cpp)
+  add_subdirectory(${yaml-cpp_SOURCE_DIR} ${yaml-cpp_BINARY_DIR})
+endif()
+
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")
 
 include_directories(${TORCH_INCLUDE_DIRS})
diff --git a/cpp/build.sh b/cpp/build.sh
index bec3e04233..b273255b6a 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -124,37 +124,6 @@ function install_libtorch() {
   cd "$BWD" || exit
 }
 
-function install_yaml_cpp() {
-  YAML_CPP_SRC_DIR=$BASE_DIR/third-party/yaml-cpp
-  YAML_CPP_BUILD_DIR=$DEPS_DIR/yaml-cpp-build
-
-  if [ ! -d "$YAML_CPP_SRC_DIR" ] ; then
-    echo -e "${COLOR_GREEN}[ INFO ] Cloning yaml-cpp repo ${COLOR_OFF}"
-    git clone https://github.com/jbeder/yaml-cpp.git "$YAML_CPP_SRC_DIR"
-    cd $YAML_CPP_SRC_DIR
-    git checkout tags/0.8.0
-  fi
-
-  if [ ! -d "$YAML_CPP_BUILD_DIR" ] ; then
-    echo -e "${COLOR_GREEN}[ INFO ] Building yaml-cpp ${COLOR_OFF}"
-
-    if [ "$PLATFORM" = "Linux" ]; then
-      SUDO="sudo"
-    elif [ "$PLATFORM" = "Mac" ]; then
-      SUDO=""
-    fi
-
-    mkdir $YAML_CPP_BUILD_DIR
-    cd $YAML_CPP_BUILD_DIR
-    cmake $YAML_CPP_SRC_DIR -DYAML_BUILD_SHARED_LIBS=ON -DYAML_CPP_BUILD_TESTS=OFF -DCMAKE_CXX_FLAGS="-fPIC"
-    $SUDO make install
-
-    echo -e "${COLOR_GREEN}[ INFO ] yaml-cpp is installed ${COLOR_OFF}"
-  fi
-
-  cd "$BWD" || exit
-}
-
 function build_llama_cpp() {
   BWD=$(pwd)
   LLAMA_CPP_SRC_DIR=$BASE_DIR/third-party/llama.cpp
@@ -288,14 +257,6 @@ function symlink_torch_libs() {
   fi
 }
 
-function symlink_yaml_cpp_lib() {
-  if [ "$PLATFORM" = "Linux" ]; then
-    ln -sf ${DEPS_DIR}/yaml-cpp-build/*.so* ${LIBS_DIR}
-  elif [ "$PLATFORM" = "Mac" ]; then
-    ln -sf ${DEPS_DIR}/yaml-cpp-build/*.dylib* ${LIBS_DIR}
-  fi
-}
-
 function install_torchserve_cpp() {
   TARGET_DIR=$BASE_DIR/../ts/cpp/
 
@@ -378,10 +339,8 @@ git submodule update --init --recursive
 install_folly
 install_kineto
 install_libtorch
-install_yaml_cpp
 build_llama_cpp
 prepare_test_files
 build
 symlink_torch_libs
-symlink_yaml_cpp_lib
 install_torchserve_cpp

From 9e777ba5a90471774f22af10de099f9b72b5c026 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 22 Feb 2024 21:32:37 +0000
Subject: [PATCH 05/47] move cpp binaries to location based on ts.__file__

---
 cpp/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/build.sh b/cpp/build.sh
index b273255b6a..a2c241ef3a 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -258,7 +258,7 @@ function symlink_torch_libs() {
 }
 
 function install_torchserve_cpp() {
-  TARGET_DIR=$BASE_DIR/../ts/cpp/
+  TARGET_DIR=`python -c "import ts; from pathlib import Path; print(Path(ts.__file__).parent / 'cpp')"`
 
   if [ -d $TARGET_DIR ]; then
     rm -rf $TARGET_DIR

From 72ddc3eb168c7b172885c18ea3567260081655a2 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 22 Feb 2024 21:32:59 +0000
Subject: [PATCH 06/47] Add FAQ to cpp readme

---
 cpp/README.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/cpp/README.md b/cpp/README.md
index 42df03fbd1..8b006fb52a 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -66,3 +66,13 @@ This will make the TorchServe installation editable and the updated cpp backend
 ```
 python ts_scripts/install_from_src.py --environment dev
 ```
+
+#### FAQs
+Q: After running ./build.sh TorchServe can not find model_worker_socket
+A:
+1. See if the binary `model_worker_socket` exists by running:
+```bash
+python -c "import ts; from pathlib import Path; print((Path(ts.__file__).parent / 'cpp/bin/model_worker_socket').exists())
+```
+2. Look if ./build.sh was actually successful and if the tests ran without any error at the end. If a test failed the binary will not be copied into the appropriate directory.
+3. Make sure you have the right conda/venv environment activated during building that you're also using to run TorchServe.

From 00c862704a2aa10eb0fcbc86baa58a156f754749 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 22 Feb 2024 21:37:16 +0000
Subject: [PATCH 07/47] Update readme to expect installed TS

---
 cpp/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cpp/README.md b/cpp/README.md
index 8b006fb52a..c169d066b1 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -4,20 +4,20 @@
 * GCC version: gcc-9
 ## Installation and Running TorchServe CPP
 
+This installation instruction assumes that TorchServe is already installed through pip/conda/source. If this is not the case install it after the `Install dependencies` step through your preferred method.
+
 ### Install dependencies
 ```
 cd serve
 python ts_scripts/install_dependencies.py --cpp --environment dev [--cuda=cu121|cu118]
 ```
 ### Building the backend
+Don't forget to install or update TorchServe at this point if it wasn't previously installed.
 ```
 ## Dev Build
 cd cpp
 ./build.sh [-g cu121|cu118]
 
-## Install TorchServe from source
-cd ..
-python ts_scripts/install_from_src.py
 ```
 ### Run TorchServe
 ```

From 2875fbf25a237f82a80d4ef47b7379d636181e8a Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 22 Feb 2024 22:04:28 +0000
Subject: [PATCH 08/47] Fix libtorch install on mac

---
 cpp/build.sh | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/cpp/build.sh b/cpp/build.sh
index a2c241ef3a..45c088da72 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -74,7 +74,17 @@ function install_kineto() {
 }
 
 function install_libtorch() {
+  cd "$DEPS_DIR" || exit
   TORCH_VERSION="2.2.0"
+  if [ -d "$DEPS_DIR/libtorch" ]; then
+    RAW_VERSION=`cat "$DEPS_DIR/libtorch/build-version"`
+    VERSION=`cat "$DEPS_DIR/libtorch/build-version" | cut -d "+" -f 1`
+    if [ "$USE_NIGHTLIES" = "true" ] && [[ ! "${RAW_VERSION}" =~ .*"dev".* ]]; then
+      rm -rf "$DEPS_DIR/libtorch"
+    elif [ "$USE_NIGHTLIES" == "" ] && [ "$VERSION" != "$TORCH_VERSION" ]; then
+      rm -rf "$DEPS_DIR/libtorch"
+    fi
+  fi
   if [ "$PLATFORM" = "Mac" ]; then
     if [ ! -d "$DEPS_DIR/libtorch" ]; then
       if [[ $(uname -m) == 'x86_64' ]]; then
@@ -95,17 +105,7 @@ function install_libtorch() {
       echo -e "${COLOR_RED}[ ERROR ] Unknown platform: $PLATFORM ${COLOR_OFF}"
       exit 1
   else  # Linux
-    if [ -d "$DEPS_DIR/libtorch" ]; then
-      RAW_VERSION=`cat "$DEPS_DIR/libtorch/build-version"`
-      VERSION=`cat "$DEPS_DIR/libtorch/build-version" | cut -d "+" -f 1`
-      if [ "$USE_NIGHTLIES" = "true" ] && [[ ! "${RAW_VERSION}" =~ .*"dev".* ]]; then
-        rm -rf "$DEPS_DIR/libtorch"
-      elif [ "$USE_NIGHTLIES" == "" ] && [ "$VERSION" != "$TORCH_VERSION" ]; then
-        rm -rf "$DEPS_DIR/libtorch"
-      fi
-    fi
     if [ ! -d "$DEPS_DIR/libtorch" ]; then
-      cd "$DEPS_DIR" || exit
       echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Linux ${COLOR_OFF}"
       if [ "$USE_NIGHTLIES" == true ]; then
         URL=https://download.pytorch.org/libtorch/nightly/${CUDA}/libtorch-cxx11-abi-shared-with-deps-latest.zip
@@ -196,12 +196,11 @@ function build() {
 
   # Build torchserve_cpp with cmake
   cd "$BWD" || exit
-  YAML_CPP_CMAKE_DIR=$DEPS_DIR/yaml-cpp-build
   FOLLY_CMAKE_DIR=$DEPS_DIR/folly-build/installed
   find $FOLLY_CMAKE_DIR -name "lib*.*"  -exec ln -s "{}" $LIBS_DIR/ \;
   if [ "$PLATFORM" = "Linux" ]; then
     cmake                                                                                     \
-    -DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$YAML_CPP_CMAKE_DIR;$DEPS_DIR/libtorch"                       \
+    -DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$DEPS_DIR/libtorch"                       \
     -DCMAKE_INSTALL_PREFIX="$PREFIX"                                                          \
     "$MAYBE_BUILD_QUIC"                                                                       \
     "$MAYBE_BUILD_TESTS"                                                                      \
@@ -218,7 +217,7 @@ function build() {
     fi
   elif [ "$PLATFORM" = "Mac" ]; then
     cmake                                                                                     \
-    -DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$YAML_CPP_CMAKE_DIR;$DEPS_DIR/libtorch"    \
+    -DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$DEPS_DIR/libtorch"                       \
     -DCMAKE_INSTALL_PREFIX="$PREFIX"                                                          \
     "$MAYBE_BUILD_QUIC"                                                                       \
     "$MAYBE_BUILD_TESTS"                                                                      \

From ff432ba0ec6ca7940b7e4e395c3827ca0cb91ffc Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 22 Feb 2024 21:03:29 -0800
Subject: [PATCH 09/47] Add cpp build faqs

---
 cpp/README.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cpp/README.md b/cpp/README.md
index c169d066b1..0bf1076c96 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -76,3 +76,10 @@ python -c "import ts; from pathlib import Path; print((Path(ts.__file__).parent
 ```
 2. Look if ./build.sh was actually successful and if the tests ran without any error at the end. If a test failed the binary will not be copied into the appropriate directory.
 3. Make sure you have the right conda/venv environment activated during building that you're also using to run TorchServe.
+
+Q: Build on Mac fails with `Library not loaded: @rpath/libomp.dylib`
+A:  Install libomp with brew and link in /usr/local/lib 
+```bash
+brew install libomp
+sudo ln -s /opt/homebrew/opt/libomp/lib/libomp.dylib /usr/local/lib/libomp.dylib
+```
\ No newline at end of file

From 4da43aee69e177adf182012594f688f6ed5727d7 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 22 Feb 2024 21:07:44 -0800
Subject: [PATCH 10/47] Remove llama.cpp as submodule and add through
 fetch_content

---
 .gitmodules                                   |  3 ---
 cpp/build.sh                                  | 17 +++----------
 cpp/third-party/llama.cpp                     |  1 -
 examples/cpp/llamacpp/CMakeLists.txt          | 25 +++++++++----------
 examples/cpp/llamacpp/src/llamacpp_handler.cc |  2 +-
 ts_scripts/install_dependencies.py            |  2 ++
 6 files changed, 18 insertions(+), 32 deletions(-)
 delete mode 160000 cpp/third-party/llama.cpp

diff --git a/.gitmodules b/.gitmodules
index 3125a3b997..5099aedbd3 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,9 +1,6 @@
 [submodule "third_party/google/rpc"]
 	path = third_party/google/rpc
 	url = https://github.com/googleapis/googleapis.git
-[submodule "cpp/third-party/llama.cpp"]
-	path = cpp/third-party/llama.cpp
-	url = https://github.com/ggerganov/llama.cpp.git
 [submodule "cpp/third-party/llama2.c"]
 	path = cpp/third-party/llama2.c
 	url = https://github.com/karpathy/llama2.c
diff --git a/cpp/build.sh b/cpp/build.sh
index 45c088da72..88d03989d5 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -124,18 +124,6 @@ function install_libtorch() {
   cd "$BWD" || exit
 }
 
-function build_llama_cpp() {
-  BWD=$(pwd)
-  LLAMA_CPP_SRC_DIR=$BASE_DIR/third-party/llama.cpp
-  cd "${LLAMA_CPP_SRC_DIR}"
-  if [ "$PLATFORM" = "Mac" ]; then
-    make LLAMA_METAL=OFF -j
-  else
-    make -j
-  fi
-  cd "$BWD" || exit
-}
-
 function prepare_test_files() {
   echo -e "${COLOR_GREEN}[ INFO ]Preparing test files ${COLOR_OFF}"
   local EX_DIR="${TR_DIR}/examples/"
@@ -216,6 +204,8 @@ function build() {
       export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/bin/nvcc
     fi
   elif [ "$PLATFORM" = "Mac" ]; then
+    export LIBRARY_PATH=${LIBRARY_PATH}:`brew --prefix icu4c`/lib:`brew --prefix libomp`/lib
+
     cmake                                                                                     \
     -DCMAKE_PREFIX_PATH="$DEPS_DIR;$FOLLY_CMAKE_DIR;$DEPS_DIR/libtorch"                       \
     -DCMAKE_INSTALL_PREFIX="$PREFIX"                                                          \
@@ -228,7 +218,7 @@ function build() {
     "$MAYBE_NIGHTLIES"                                                                        \
     ..
 
-    export LIBRARY_PATH=${LIBRARY_PATH}:/usr/local/opt/icu4c/lib
+    
   else
     # TODO: Windows
     echo -e "${COLOR_RED}[ ERROR ] Unknown platform: $PLATFORM ${COLOR_OFF}"
@@ -338,7 +328,6 @@ git submodule update --init --recursive
 install_folly
 install_kineto
 install_libtorch
-build_llama_cpp
 prepare_test_files
 build
 symlink_torch_libs
diff --git a/cpp/third-party/llama.cpp b/cpp/third-party/llama.cpp
deleted file mode 160000
index cd4fddb29f..0000000000
--- a/cpp/third-party/llama.cpp
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit cd4fddb29f81d6a1f6d51a0c016bc6b486d68def
diff --git a/examples/cpp/llamacpp/CMakeLists.txt b/examples/cpp/llamacpp/CMakeLists.txt
index e071167585..aa9e7d1682 100644
--- a/examples/cpp/llamacpp/CMakeLists.txt
+++ b/examples/cpp/llamacpp/CMakeLists.txt
@@ -2,19 +2,18 @@ set(LLAMACPP_SRC_DIR "${torchserve_cpp_SOURCE_DIR}/third-party/llama.cpp")
 
 add_library(llamacpp_handler SHARED src/llamacpp_handler.cc)
 
-set(MY_OBJECT_FILES
-    ${LLAMACPP_SRC_DIR}/ggml.o
-    ${LLAMACPP_SRC_DIR}/llama.o
-    ${LLAMACPP_SRC_DIR}/common.o
-    ${LLAMACPP_SRC_DIR}/ggml-quants.o
-    ${LLAMACPP_SRC_DIR}/ggml-alloc.o
-    ${LLAMACPP_SRC_DIR}/grammar-parser.o
-    ${LLAMACPP_SRC_DIR}/console.o
-    ${LLAMACPP_SRC_DIR}/build-info.o
-    ${LLAMACPP_SRC_DIR}/ggml-backend.o
-
+FetchContent_Declare(
+  llama.cpp
+  GIT_REPOSITORY https://github.com/ggerganov/llama.cpp
+  GIT_TAG b2241
 )
+FetchContent_GetProperties(llama.cpp)
+
+if(NOT llama.cpp_POPULATED)
+  message(STATUS "Fetching llama.cpp...")
+  FetchContent_Populate(llama.cpp)
+  add_subdirectory(${llama.cpp_SOURCE_DIR} ${llama.cpp_BINARY_DIR})
+endif()
 
-target_sources(llamacpp_handler PRIVATE ${MY_OBJECT_FILES})
 target_include_directories(llamacpp_handler PUBLIC ${LLAMACPP_SRC_DIR})
-target_link_libraries(llamacpp_handler PRIVATE ts_backends_core ts_utils ${TORCH_LIBRARIES})
+target_link_libraries(llamacpp_handler PRIVATE ts_backends_core ts_utils ${TORCH_LIBRARIES} common llama)
diff --git a/examples/cpp/llamacpp/src/llamacpp_handler.cc b/examples/cpp/llamacpp/src/llamacpp_handler.cc
index 6917ee44cf..ce5703a9be 100644
--- a/examples/cpp/llamacpp/src/llamacpp_handler.cc
+++ b/examples/cpp/llamacpp/src/llamacpp_handler.cc
@@ -44,7 +44,7 @@ LlamaCppHandler::LoadModel(
     params.main_gpu = 0;
     params.n_gpu_layers = 35;
 
-    llama_backend_init(params.numa);
+    llama_backend_init();
     ctx_params = llama_context_default_params();
     model_params = llama_model_default_params();
     llamamodel = llama_load_model_from_file(params.model.c_str(), model_params);
diff --git a/ts_scripts/install_dependencies.py b/ts_scripts/install_dependencies.py
index 7b31664474..fda4195dc0 100644
--- a/ts_scripts/install_dependencies.py
+++ b/ts_scripts/install_dependencies.py
@@ -60,6 +60,8 @@
     "openssl",
     "libsodium",
     "llv",
+    "icu4c",
+    "libomp",
 )
 
 CPP_DARWIN_DEPENDENCIES_LINK = (

From 11377a0961ef82c4b719801aac0fb17dffc6d875 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 13:51:31 -0800
Subject: [PATCH 11/47] Disable metal in llamacpp example

---
 cpp/build.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cpp/build.sh b/cpp/build.sh
index 88d03989d5..4e2e716190 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -216,6 +216,7 @@ function build() {
     "$MAYBE_USE_STATIC_DEPS"                                                                  \
     "$MAYBE_LIB_FUZZING_ENGINE"                                                               \
     "$MAYBE_NIGHTLIES"                                                                        \
+    "-DLLAMA_METAL=OFF"                                                                       \
     ..
 
     

From 114bf43454fb24097c296e12f57ea3466aa0997c Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 14:12:00 -0800
Subject: [PATCH 12/47] Fix dangling pointer error in cpp worker

---
 cpp/src/backends/process/model_worker.cc | 31 ++++++++++++------------
 cpp/src/backends/process/model_worker.hh |  2 +-
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/cpp/src/backends/process/model_worker.cc b/cpp/src/backends/process/model_worker.cc
index ae6afaeb01..475629c180 100644
--- a/cpp/src/backends/process/model_worker.cc
+++ b/cpp/src/backends/process/model_worker.cc
@@ -34,13 +34,13 @@ void SocketServer::Initialize(
       if (port_num.empty()) {
         TS_LOG(FATAL, "Wrong arguments passed. No socket port given.");
       }
-      port_ = htons(stoi(port_num));
+      port_ = stoi(port_num);
     }
   } else {
     TS_LOG(FATAL, "Incomplete data provided");
   }
 
-  TS_LOGF(INFO, "Listening on port: {}", socket_name);
+  TS_LOGF(INFO, "Listening on {}", socket_name_);
   server_socket_ = socket(socket_family, SOCK_STREAM, 0);
   if (server_socket_ == -1) {
     TS_LOGF(FATAL, "Failed to create socket descriptor. errno: {}", errno);
@@ -60,24 +60,25 @@ void SocketServer::Run() {
   }
 
   sockaddr *srv_sock_address = nullptr, client_sock_address{};
+  sockaddr_un sock_addr_un{};
+  sockaddr_in sock_addr_in{};
   socklen_t name_len = 0;
   if (socket_type_ == "unix") {
     TS_LOG(INFO, "Binding to unix socket");
-    sockaddr_un sock_addr{};
-    std::memset(&sock_addr, 0, sizeof(sock_addr));
-    sock_addr.sun_family = AF_UNIX;
-    std::strncpy(sock_addr.sun_path, socket_name_.c_str(),
-                 sizeof(sock_addr.sun_path));
-    srv_sock_address = reinterpret_cast<sockaddr*>(&sock_addr);
-    name_len = SUN_LEN(&sock_addr);
+    std::memset(&sock_addr_un, 0, sizeof(sock_addr_un));
+    sock_addr_un.sun_family = AF_UNIX;
+    std::strncpy(sock_addr_un.sun_path, socket_name_.c_str(),
+                 sizeof(sock_addr_un.sun_path));
+    srv_sock_address = reinterpret_cast<sockaddr*>(&sock_addr_un);
+    name_len = SUN_LEN(&sock_addr_un);
   } else {
     TS_LOG(INFO, "Binding to tcp socket");
-    sockaddr_in sock_addr{};
-    std::memset(&sock_addr, 0, sizeof(sock_addr));
-    sock_addr.sin_family = AF_INET;
-    sock_addr.sin_port = port_;
-    sock_addr.sin_addr.s_addr = inet_addr(socket_name_.c_str());
-    srv_sock_address = reinterpret_cast<sockaddr*>(&sock_addr);
+    std::memset(&sock_addr_in, 0, sizeof(sock_addr_in));
+    sock_addr_in.sin_family = AF_INET;
+    TS_LOGF(INFO, "Listening on port {}", port_);
+    sock_addr_in.sin_port = htons(port_);
+    sock_addr_in.sin_addr.s_addr = inet_addr(socket_name_.c_str());
+    srv_sock_address = reinterpret_cast<sockaddr*>(&sock_addr_in);
     name_len = sizeof(*srv_sock_address);
   }
 
diff --git a/cpp/src/backends/process/model_worker.hh b/cpp/src/backends/process/model_worker.hh
index 0448fc7238..20ed457cff 100644
--- a/cpp/src/backends/process/model_worker.hh
+++ b/cpp/src/backends/process/model_worker.hh
@@ -57,7 +57,7 @@ class SocketServer {
   int server_socket_ = -1;
   std::string socket_type_;
   std::string socket_name_;
-  ushort port_ = 9000;
+  int port_ = 9000;
   std::shared_ptr<torchserve::Backend> backend_;
 };
 

From f9d12a7950c80a0bf056a620a8c9ecf3b82854ba Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 14:27:10 -0800
Subject: [PATCH 13/47] Remove kineto from mac build

---
 cpp/build.sh | 26 +++-----------------------
 1 file changed, 3 insertions(+), 23 deletions(-)

diff --git a/cpp/build.sh b/cpp/build.sh
index 4e2e716190..6f9e72e347 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -54,25 +54,6 @@ function install_folly() {
   echo "$FOLLY_BUILD_DIR/installed"
 }
 
-function install_kineto() {
-  if [ "$PLATFORM" = "Linux" ]; then
-    echo -e "${COLOR_GREEN}[ INFO ] Skip install kineto on Linux ${COLOR_OFF}"
-  elif [ "$PLATFORM" = "Mac" ]; then
-    KINETO_SRC_DIR=$BASE_DIR/third-party/kineto
-
-    if [ ! -d "$KINETO_SRC_DIR" ] ; then
-      echo -e "${COLOR_GREEN}[ INFO ] Cloning kineto repo ${COLOR_OFF}"
-      git clone --recursive https://github.com/pytorch/kineto.git "$KINETO_SRC_DIR"
-      cd $KINETO_SRC_DIR/libkineto
-      mkdir build && cd build
-      cmake ..
-      make install
-    fi
-  fi
-
-  cd "$BWD" || exit
-}
-
 function install_libtorch() {
   cd "$DEPS_DIR" || exit
   TORCH_VERSION="2.2.0"
@@ -90,12 +71,12 @@ function install_libtorch() {
       if [[ $(uname -m) == 'x86_64' ]]; then
         echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Mac x86_64 ${COLOR_OFF}"
         wget https://download.pytorch.org/libtorch/cpu/libtorch-macos-x86_64-${TORCH_VERSION}.zip
-        unzip libtorch-macos-x86_64-${TORCH_VERSION}.zip
+        unzip -q libtorch-macos-x86_64-${TORCH_VERSION}.zip
         rm libtorch-macos-x86_64-${TORCH_VERSION}.zip
       else
         echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Mac arm64 ${COLOR_OFF}"
         wget https://download.pytorch.org/libtorch/cpu/libtorch-macos-arm64-${TORCH_VERSION}.zip
-        unzip libtorch-macos-arm64-${TORCH_VERSION}.zip
+        unzip -q libtorch-macos-arm64-${TORCH_VERSION}.zip
         rm libtorch-macos-arm64-${TORCH_VERSION}.zip
       fi
     fi
@@ -115,7 +96,7 @@ function install_libtorch() {
       wget $URL
       ZIP_FILE=$(basename "$URL")
       ZIP_FILE="${ZIP_FILE//%2B/+}"
-      unzip $ZIP_FILE
+      unzip -q $ZIP_FILE
       rm $ZIP_FILE
     fi
     echo -e "${COLOR_GREEN}[ INFO ] libtorch is installed ${COLOR_OFF}"
@@ -327,7 +308,6 @@ cd $BASE_DIR
 git submodule update --init --recursive
 
 install_folly
-install_kineto
 install_libtorch
 prepare_test_files
 build

From 03081895bdf5b2bf93b35cedde4de61d55096d8d Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 15:48:02 -0800
Subject: [PATCH 14/47] Add llvm as mac cpp dependencies (req for clang-tidy)

---
 ts_scripts/install_dependencies.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ts_scripts/install_dependencies.py b/ts_scripts/install_dependencies.py
index 4be538c264..7be1c47a65 100644
--- a/ts_scripts/install_dependencies.py
+++ b/ts_scripts/install_dependencies.py
@@ -62,6 +62,7 @@
     "llv",
     "icu4c",
     "libomp",
+    "llvm",
 )
 
 CPP_DARWIN_DEPENDENCIES_LINK = (

From 0a599476fe3824ae291edc09dafd2249c3ab7463 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 16:00:45 -0800
Subject: [PATCH 15/47] Enable fPIC for llama.cpp example

---
 examples/cpp/llamacpp/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/examples/cpp/llamacpp/CMakeLists.txt b/examples/cpp/llamacpp/CMakeLists.txt
index aa9e7d1682..084c667830 100644
--- a/examples/cpp/llamacpp/CMakeLists.txt
+++ b/examples/cpp/llamacpp/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLAMACPP_SRC_DIR "${torchserve_cpp_SOURCE_DIR}/third-party/llama.cpp")
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
 add_library(llamacpp_handler SHARED src/llamacpp_handler.cc)
 

From 563c2da59c885b4d94839c18c9d9802def8ac764 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 16:26:39 -0800
Subject: [PATCH 16/47] Add sudo to clang-tidy link creation

---
 ts_scripts/install_dependencies.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ts_scripts/install_dependencies.py b/ts_scripts/install_dependencies.py
index 7be1c47a65..5da661ed8a 100644
--- a/ts_scripts/install_dependencies.py
+++ b/ts_scripts/install_dependencies.py
@@ -284,13 +284,13 @@ def install_cpp_dependencies(self):
             os.system(f"brew install -f {' '.join(CPP_DARWIN_DEPENDENCIES)}")
             os.system(f"brew link {' '.join(CPP_DARWIN_DEPENDENCIES_LINK)}")
             os.system(
-                'ln -s "$(brew --prefix llvm)/bin/clang-format" "/usr/local/bin/clang-format"'
+                f'{self.sudo_cmd} ln -s "$(brew --prefix llvm)/bin/clang-format" "/usr/local/bin/clang-format"'
             )
             os.system(
-                'ln -s "$(brew --prefix llvm)/bin/clang-tidy" "/usr/local/bin/clang-tidy"'
+                f'{self.sudo_cmd} ln -s "$(brew --prefix llvm)/bin/clang-tidy" "/usr/local/bin/clang-tidy"'
             )
             os.system(
-                'ln -s "$(brew --prefix llvm)/bin/clang-apply-replacements" "/usr/local/bin/clang-apply-replacements"'
+                f'{self.sudo_cmd} ln -s "$(brew --prefix llvm)/bin/clang-apply-replacements" "/usr/local/bin/clang-apply-replacements"'
             )
 
     def install_neuronx_driver(self):

From b4c95dbd413846649598dc4ff0bd76d23eaf3ffb Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 16:42:00 -0800
Subject: [PATCH 17/47] Add undefined symbol faq

---
 cpp/README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cpp/README.md b/cpp/README.md
index 0bf1076c96..cbc627ddae 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -78,8 +78,11 @@ python -c "import ts; from pathlib import Path; print((Path(ts.__file__).parent
 3. Make sure you have the right conda/venv environment activated during building that you're also using to run TorchServe.
 
 Q: Build on Mac fails with `Library not loaded: @rpath/libomp.dylib`
-A:  Install libomp with brew and link in /usr/local/lib 
+A: Install libomp with brew and link in /usr/local/lib 
 ```bash
 brew install libomp
 sudo ln -s /opt/homebrew/opt/libomp/lib/libomp.dylib /usr/local/lib/libomp.dylib
-```
\ No newline at end of file
+```
+
+Q: When loading a handler which uses a model exported with torch._export.aot_compile the handler dies with "error: Error in dlopen: MODEL.SO : undefined symbol: SOME_SYMBOL".
+A: Make sure that you are using matching libtorch and Pytorch versions for inference and export, respectively.
\ No newline at end of file

From e5379a7b09d3675b1743f595096f5c042a9b0074 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 16:46:33 -0800
Subject: [PATCH 18/47] fix llv typo

---
 ts_scripts/install_dependencies.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ts_scripts/install_dependencies.py b/ts_scripts/install_dependencies.py
index 5da661ed8a..d8d6bfb134 100644
--- a/ts_scripts/install_dependencies.py
+++ b/ts_scripts/install_dependencies.py
@@ -59,7 +59,6 @@
     "xz",
     "openssl",
     "libsodium",
-    "llv",
     "icu4c",
     "libomp",
     "llvm",

From 03ba212eeeb9c28c2a19f294a2f6781d3cd1e0ae Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 20:36:02 -0800
Subject: [PATCH 19/47] Add install from source to cpp_ci

---
 .github/workflows/ci-cpu-cpp.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 73f3b15335..1dee62c356 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -29,6 +29,9 @@ jobs:
       - name: Install dependencies
         run: |
           python ts_scripts/install_dependencies.py --environment=dev --cpp
+      - name: Install from source
+        run: |
+          python ts_scripts/install_from_source.py
       - name: Build
         run: |
           cd cpp && ./build.sh

From 17a0da3708a8a6f7c84de1c3e2bb6e3343e35311 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 20:40:24 -0800
Subject: [PATCH 20/47] Correct install from source to cpp_ci

---
 .github/workflows/ci-cpu-cpp.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 1dee62c356..3a708f0751 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -29,12 +29,9 @@ jobs:
       - name: Install dependencies
         run: |
           python ts_scripts/install_dependencies.py --environment=dev --cpp
-      - name: Install from source
+      - name: Install TorchServe
         run: |
-          python ts_scripts/install_from_source.py
+          python ts_scripts/install_from_src.py
       - name: Build
         run: |
           cd cpp && ./build.sh
-      - name: Install TorchServe
-        run: |
-          cd .. && python ts_scripts/install_from_src.py

From ef62412106ef8bfac25123bcdfab57a4f70e0c80 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 21:29:31 -0800
Subject: [PATCH 21/47] bump up pyyaml version to avoid cython 3 issue
 https://github.com/yaml/pyyaml/pull/702

---
 requirements/common.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/common.txt b/requirements/common.txt
index 2023978013..b07ff9637b 100644
--- a/requirements/common.txt
+++ b/requirements/common.txt
@@ -3,5 +3,5 @@ requests==2.31.0
 captum==0.6.0
 packaging==23.2
 pynvml==11.5.0
-pyyaml==6.0
+pyyaml==6.0.1
 ninja==1.11.1.1

From d9d338b4cf77b565d76d10b9841c45ab47058ceb Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Mon, 26 Feb 2024 21:46:44 -0800
Subject: [PATCH 22/47] Move cpp ci to M1 mac

---
 .github/workflows/ci-cpu-cpp.yml | 18 +++++++++++++++++-
 requirements/common.txt          |  2 +-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 3a708f0751..80ec1f6e11 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -16,8 +16,24 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macOS-latest]
+        os: [ubuntu-20.04, macos-14]
     steps:
+      - name: Setup Python for M1
+        if: matrix.os == 'macos-14'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Setup Python for all other OS
+        if: matrix.os != 'macos-14'
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+          architecture: x64
+      - name: Setup Java 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '17'
       - name: Checkout TorchServe
         uses: actions/checkout@v3
         with:
diff --git a/requirements/common.txt b/requirements/common.txt
index b07ff9637b..2023978013 100644
--- a/requirements/common.txt
+++ b/requirements/common.txt
@@ -3,5 +3,5 @@ requests==2.31.0
 captum==0.6.0
 packaging==23.2
 pynvml==11.5.0
-pyyaml==6.0.1
+pyyaml==6.0
 ninja==1.11.1.1

From 18a1164b842c36e872676610b7fe5a958e9a4862 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 27 Feb 2024 09:51:12 -0800
Subject: [PATCH 23/47] Run workflow on self hosted runner

---
 .github/workflows/ci-cpu-cpp.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 80ec1f6e11..3d36617b01 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -1,5 +1,6 @@
 name: CI CPU - CPP Build
 
+runs-on: self-hosted
 on:
   workflow_dispatch:
   push:

From 1537eb6fc132d2e8e84e9149ed927e651f1aae34 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 27 Feb 2024 09:51:47 -0800
Subject: [PATCH 24/47] Disable mac ci for cpp

---
 .github/workflows/ci-cpu-cpp.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 3d36617b01..bc5b0c0fb6 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -17,7 +17,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-14]
+        os: [ubuntu-20.04]
+        # os: [ubuntu-20.04, macos-14]
     steps:
       - name: Setup Python for M1
         if: matrix.os == 'macos-14'

From e6ff2accb4712146ed40137a7d49dbef3d523343 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 27 Feb 2024 09:53:59 -0800
Subject: [PATCH 25/47] Fix workflow syntax

---
 .github/workflows/ci-cpu-cpp.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index bc5b0c0fb6..a186626afa 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -1,6 +1,5 @@
 name: CI CPU - CPP Build
 
-runs-on: self-hosted
 on:
   workflow_dispatch:
   push:
@@ -13,7 +12,8 @@ concurrency:
 
 jobs:
   ci-cpu-cpp:
-    runs-on: ${{ matrix.os }}
+    # runs-on: ${{ matrix.os }}
+    runs-on: self-hosted
     strategy:
       fail-fast: false
       matrix:

From ae31141ac051b336498c8bdfe65f28955900f92f Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 27 Feb 2024 09:56:40 -0800
Subject: [PATCH 26/47] Run on cpp-ci

---
 .github/workflows/ci-cpu-cpp.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index a186626afa..00e913db87 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -13,7 +13,7 @@ concurrency:
 jobs:
   ci-cpu-cpp:
     # runs-on: ${{ matrix.os }}
-    runs-on: self-hosted
+    runs-on: ci-cpp
     strategy:
       fail-fast: false
       matrix:

From 5faba3e9464280bf0389b82b808a67e58318578b Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 27 Feb 2024 11:23:58 -0800
Subject: [PATCH 27/47] Remove sudo

---
 cpp/build.sh | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/cpp/build.sh b/cpp/build.sh
index 673115280a..f20809ed5b 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -35,12 +35,7 @@ function install_folly() {
     echo -e "${COLOR_GREEN}[ INFO ] Building Folly ${COLOR_OFF}"
     cd $FOLLY_SRC_DIR
 
-    if [ "$PLATFORM" = "Linux" ]; then
-      SUDO="sudo"
-    elif [ "$PLATFORM" = "Mac" ]; then
-      SUDO=""
-    fi
-    $SUDO ./build/fbcode_builder/getdeps.py install-system-deps --recursive
+    ./build/fbcode_builder/getdeps.py install-system-deps --recursive
 
     ./build/fbcode_builder/getdeps.py build \
     --allow-system-packages \

From da760498f6f49febd4e67ebf7343e0b5519883e9 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 27 Feb 2024 14:44:03 -0800
Subject: [PATCH 28/47] Add apt update for act docker

---
 .github/workflows/ci-cpu-cpp.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 00e913db87..9277b4a4a0 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -46,7 +46,7 @@ jobs:
             brew install libtorch
       - name: Install dependencies
         run: |
-          python ts_scripts/install_dependencies.py --environment=dev --cpp
+          sudo apt update && python ts_scripts/install_dependencies.py --environment=dev --cpp
       - name: Install TorchServe
         run: |
           python ts_scripts/install_from_src.py

From 966fc9e7ef646ebb9c4c44643657b0735731193e Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 27 Feb 2024 15:50:13 -0800
Subject: [PATCH 29/47] print library_path in print_env_info.py

---
 ts_scripts/print_env_info.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/ts_scripts/print_env_info.py b/ts_scripts/print_env_info.py
index afb71b6660..4a4a64508c 100644
--- a/ts_scripts/print_env_info.py
+++ b/ts_scripts/print_env_info.py
@@ -43,6 +43,8 @@
 
 npm_env = {"npm_pkg_version": []}
 
+cpp_env = {"LIBRARY_PATH" : ""}
+
 
 def get_nvidia_smi():
     # Note: nvidia-smi is currently available only on Windows and Linux
@@ -284,6 +286,16 @@ def get_torch_model_archiver():
     return version
 
 
+def get_library_path():
+    platform = get_platform()
+    if platform == "darwin":
+        return os.environ.get("DYLD_LIBRARY_PATH", "")
+    elif platform == "linux":
+        return os.environ.get("LD_LIBRARY_PATH", "")
+    else:
+        return ""
+
+
 def populate_torchserve_env(torch_pkg):
     for pkg in torch_pkg:
         if pkg.split("==")[0] == "torch":
@@ -336,6 +348,9 @@ def populate_cuda_env(cuda_available_str):
 
 def populate_npm_env():
     npm_env["npm_pkg_version"] = get_npm_packages()
+    
+def populate_cpp_env():
+    cpp_env["LIBRARY_PATH"] = get_library_path()
 
 
 def populate_env_info():
@@ -360,6 +375,9 @@ def populate_env_info():
 
     if get_platform() == "darwin":
         populate_npm_env()
+        
+    if get_platform() in ("darwin", "linux"):
+        populate_cpp_env()
 
 
 env_info_fmt = """
@@ -403,11 +421,17 @@ def populate_env_info():
 {npm_pkg_version}
 """
 
+cpp_env_info_fmt = """
+Environment:
+library_path (LD_/DYLD_): {LIBRARY_PATH}
+"""
+
 
 def get_pretty_env_info(branch_name):
     global env_info_fmt
     global cuda_info_fmt
     global npm_info_fmt
+    global cpp_env_info_fmt
     populate_env_info()
     env_dict = {
         **torchserve_env,
@@ -415,6 +439,7 @@ def get_pretty_env_info(branch_name):
         **java_env,
         **os_info,
         "torchserve_branch": branch_name,
+        **cpp_env,
     }
 
     if TORCH_AVAILABLE and torch.cuda.is_available():
@@ -424,6 +449,9 @@ def get_pretty_env_info(branch_name):
     if get_platform() == "darwin":
         env_dict.update(npm_env)
         env_info_fmt = env_info_fmt + "\n" + npm_info_fmt
+        
+    if get_platform() in ("darwin", "linux"):
+        env_info_fmt = env_info_fmt + "\n" + cpp_env_info_fmt
 
     return env_info_fmt.format(**env_dict)
 

From 91665cf52f47242c2304a33745822be103efe9ca Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 27 Feb 2024 15:51:37 -0800
Subject: [PATCH 30/47] print end in cpp ci workflow

---
 .github/workflows/ci-cpu-cpp.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 9277b4a4a0..caa8fa483d 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -50,6 +50,9 @@ jobs:
       - name: Install TorchServe
         run: |
           python ts_scripts/install_from_src.py
+      - name: Print Env
+        run: |
+          python ts_scripts/print_env_info.py
       - name: Build
         run: |
           cd cpp && ./build.sh

From 5867e868cbceaacd4d5a20fa2eca534363cbe4b6 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 27 Feb 2024 16:00:45 -0800
Subject: [PATCH 31/47] Run on github runner

---
 .github/workflows/ci-cpu-cpp.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index caa8fa483d..b27bffcc2b 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -12,8 +12,8 @@ concurrency:
 
 jobs:
   ci-cpu-cpp:
-    # runs-on: ${{ matrix.os }}
-    runs-on: ci-cpp
+    runs-on: ${{ matrix.os }}
+    # runs-on: ci-cpp
     strategy:
       fail-fast: false
       matrix:

From a2f1d257edaa5ca451c3b150a0c4c14f3eb72697 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Tue, 27 Feb 2024 20:32:41 -0800
Subject: [PATCH 32/47] Add upterm session to workflow

---
 .github/workflows/ci-cpu-cpp.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index b27bffcc2b..ade6d5a1e4 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -36,6 +36,11 @@ jobs:
         with:
           distribution: 'zulu'
           java-version: '17'
+      - name: Setup upterm session
+        uses: lhotari/action-upterm@v1
+        with:
+          limit-access-to-actor: true
+          limit-access-to-users: mreso
       - name: Checkout TorchServe
         uses: actions/checkout@v3
         with:
@@ -56,3 +61,8 @@ jobs:
       - name: Build
         run: |
           cd cpp && ./build.sh
+      - name: Setup upterm session
+        uses: lhotari/action-upterm@v1
+        with:
+          limit-access-to-actor: true
+          limit-access-to-users: mreso

From f754432896a0b2f1b83b651fbea56dfa7b777595 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 29 Feb 2024 21:24:28 +0000
Subject: [PATCH 33/47] Move post mortem upterm session before build

---
 .github/workflows/ci-cpu-cpp.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index ade6d5a1e4..dd5fc6cac4 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -58,11 +58,11 @@ jobs:
       - name: Print Env
         run: |
           python ts_scripts/print_env_info.py
-      - name: Build
-        run: |
-          cd cpp && ./build.sh
       - name: Setup upterm session
         uses: lhotari/action-upterm@v1
         with:
           limit-access-to-actor: true
           limit-access-to-users: mreso
+      - name: Build
+        run: |
+          cd cpp && ./build.sh

From c17d28896b10e67ae6024a9d2b7e93f2a2569903 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 29 Feb 2024 21:30:46 +0000
Subject: [PATCH 34/47] Remove first upterm session

---
 .github/workflows/ci-cpu-cpp.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index dd5fc6cac4..bc62535c11 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -36,11 +36,6 @@ jobs:
         with:
           distribution: 'zulu'
           java-version: '17'
-      - name: Setup upterm session
-        uses: lhotari/action-upterm@v1
-        with:
-          limit-access-to-actor: true
-          limit-access-to-users: mreso
       - name: Checkout TorchServe
         uses: actions/checkout@v3
         with:

From f16f7e1845f1d98825a7337d7bcc39b12926baaa Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Thu, 29 Feb 2024 21:46:15 +0000
Subject: [PATCH 35/47] ci debugging

---
 .github/workflows/ci-cpu-cpp.yml |  6 +++---
 cpp/build.sh                     | 14 +++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index bc62535c11..42082a60fb 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -53,11 +53,11 @@ jobs:
       - name: Print Env
         run: |
           python ts_scripts/print_env_info.py
+      - name: Build
+        run: |
+          cd cpp && ./build.sh
       - name: Setup upterm session
         uses: lhotari/action-upterm@v1
         with:
           limit-access-to-actor: true
           limit-access-to-users: mreso
-      - name: Build
-        run: |
-          cd cpp && ./build.sh
diff --git a/cpp/build.sh b/cpp/build.sh
index f20809ed5b..25290703be 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -195,7 +195,7 @@ function build() {
     "-DLLAMA_METAL=OFF"                                                                       \
     ..
 
-    
+
   else
     # TODO: Windows
     echo -e "${COLOR_RED}[ ERROR ] Unknown platform: $PLATFORM ${COLOR_OFF}"
@@ -286,7 +286,7 @@ if [ ! -d "$BUILD_DIR" ] ; then
   mkdir -p $BUILD_DIR
 fi
 
-set -e nounset
+# set -e nounset
 trap 'cd $BASE_DIR' EXIT
 cd $BUILD_DIR || exit
 BWD=$(pwd)
@@ -303,8 +303,8 @@ cd $BASE_DIR
 git submodule update --init --recursive
 
 install_folly
-install_libtorch
-prepare_test_files
-build
-symlink_torch_libs
-install_torchserve_cpp
+# install_libtorch
+# prepare_test_files
+# build
+# symlink_torch_libs
+# install_torchserve_cpp

From 71ba328030b788c9452f4b8dee646c5f606b18ba Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Fri, 1 Mar 2024 18:58:38 +0000
Subject: [PATCH 36/47] Observe disk space

---
 .github/workflows/ci-cpu-cpp.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 42082a60fb..d1ac127e0f 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -44,9 +44,15 @@ jobs:
         if: matrix.os == 'macOS-latest'
         run: |
             brew install libtorch
+      - name: Free disk space
+        run: |
+          df -h
       - name: Install dependencies
         run: |
-          sudo apt update && python ts_scripts/install_dependencies.py --environment=dev --cpp
+          sudo apt update && PIP_NO_CACHE_DIR=on python ts_scripts/install_dependencies.py --environment=dev --cpp && sudo apt clean
+      - name: Free disk space
+        run: |
+          df -h
       - name: Install TorchServe
         run: |
           python ts_scripts/install_from_src.py

From 9a55564b933cffc54619e5cb7f0b38cabd9541c5 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Fri, 1 Mar 2024 19:32:10 +0000
Subject: [PATCH 37/47] move _build to /mnt on github runner

---
 .github/workflows/ci-cpu-cpp.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index d1ac127e0f..ffdb9ea281 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -61,7 +61,8 @@ jobs:
           python ts_scripts/print_env_info.py
       - name: Build
         run: |
-          cd cpp && ./build.sh
+          cd cpp && rm -rf _build && mkdir /mnt/_build && ln -s /mnt/_build _build
+          ./build.sh
       - name: Setup upterm session
         uses: lhotari/action-upterm@v1
         with:

From d90722fc0bc3aab3b4036bf760543f9535cbccc0 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Fri, 1 Mar 2024 19:36:35 +0000
Subject: [PATCH 38/47] fix permission denied

---
 .github/workflows/ci-cpu-cpp.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index ffdb9ea281..aa5899dc95 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -61,7 +61,7 @@ jobs:
           python ts_scripts/print_env_info.py
       - name: Build
         run: |
-          cd cpp && rm -rf _build && mkdir /mnt/_build && ln -s /mnt/_build _build
+          cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && ln -s /mnt/_build _build
           ./build.sh
       - name: Setup upterm session
         uses: lhotari/action-upterm@v1

From 8630fdd6695ecbc022fbbe8466fbc4c679cb6879 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Fri, 1 Mar 2024 20:40:22 +0000
Subject: [PATCH 39/47] use mount instead of ln

---
 .github/workflows/ci-cpu-cpp.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index aa5899dc95..0851080e39 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -49,7 +49,7 @@ jobs:
           df -h
       - name: Install dependencies
         run: |
-          sudo apt update && PIP_NO_CACHE_DIR=on python ts_scripts/install_dependencies.py --environment=dev --cpp && sudo apt clean
+          sudo apt update && python ts_scripts/install_dependencies.py --environment=dev --cpp
       - name: Free disk space
         run: |
           df -h
@@ -61,7 +61,7 @@ jobs:
           python ts_scripts/print_env_info.py
       - name: Build
         run: |
-          cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && ln -s /mnt/_build _build
+          cd cpp && rm -rf _build && sudo mkdir /mnt/_build && sudo chmod 777 /mnt/_build && mkdir _build && sudo mount --bind /mnt/_build _build
           ./build.sh
       - name: Setup upterm session
         uses: lhotari/action-upterm@v1

From 6ebce56b060883db07d88b296ff008ca0256323f Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Sat, 2 Mar 2024 00:48:15 +0000
Subject: [PATCH 40/47] Adjust llamacpp api

---
 examples/cpp/llamacpp/src/llamacpp_handler.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/cpp/llamacpp/src/llamacpp_handler.cc b/examples/cpp/llamacpp/src/llamacpp_handler.cc
index ce5703a9be..52593eb5c6 100644
--- a/examples/cpp/llamacpp/src/llamacpp_handler.cc
+++ b/examples/cpp/llamacpp/src/llamacpp_handler.cc
@@ -44,7 +44,7 @@ LlamaCppHandler::LoadModel(
     params.main_gpu = 0;
     params.n_gpu_layers = 35;
 
-    llama_backend_init();
+    llama_backend_init(false);
     ctx_params = llama_context_default_params();
     model_params = llama_model_default_params();
     llamamodel = llama_load_model_from_file(params.model.c_str(), model_params);

From 1a10eeceefafb03e134b8435cc6840145ed25c2d Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Sat, 2 Mar 2024 00:48:49 +0000
Subject: [PATCH 41/47] Reactivate set -e

---
 cpp/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/build.sh b/cpp/build.sh
index ee2ad686fd..40616c803c 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -289,7 +289,7 @@ if [ ! -d "$BUILD_DIR" ] ; then
   mkdir -p $BUILD_DIR
 fi
 
-# set -e nounset
+set -e nounset
 trap 'cd $BASE_DIR' EXIT
 cd $BUILD_DIR || exit
 BWD=$(pwd)

From 9e009c5eb38a40386b9103e331ee105e1c2cf9fa Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Sat, 2 Mar 2024 00:51:05 +0000
Subject: [PATCH 42/47] Remove note on env variable in cpp readme

---
 cpp/README.md | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/cpp/README.md b/cpp/README.md
index 98c8ec7ddb..e53dee0afb 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -19,15 +19,6 @@ Don't forget to install or update TorchServe at this point if it wasn't previous
 cd cpp
 ./build.sh [-g cu121|cu118]
 
-```
-### Set Environment Var
-#### On Mac
-```
-export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$(pwd)/_build/_deps/libtorch/lib
-```
-#### On Ubuntu
-```
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/_build/_deps/libtorch/lib
 ```
 
 ### Run TorchServe

From 1a55615cbe496e724c9fcc2f45bcc21901f28e29 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Sat, 2 Mar 2024 00:52:49 +0000
Subject: [PATCH 43/47] Fix linting issue in print_env_info.py

---
 ts_scripts/print_env_info.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/ts_scripts/print_env_info.py b/ts_scripts/print_env_info.py
index 4a4a64508c..0e74a61661 100644
--- a/ts_scripts/print_env_info.py
+++ b/ts_scripts/print_env_info.py
@@ -43,7 +43,7 @@
 
 npm_env = {"npm_pkg_version": []}
 
-cpp_env = {"LIBRARY_PATH" : ""}
+cpp_env = {"LIBRARY_PATH": ""}
 
 
 def get_nvidia_smi():
@@ -348,7 +348,8 @@ def populate_cuda_env(cuda_available_str):
 
 def populate_npm_env():
     npm_env["npm_pkg_version"] = get_npm_packages()
-    
+
+
 def populate_cpp_env():
     cpp_env["LIBRARY_PATH"] = get_library_path()
 
@@ -375,7 +376,7 @@ def populate_env_info():
 
     if get_platform() == "darwin":
         populate_npm_env()
-        
+
     if get_platform() in ("darwin", "linux"):
         populate_cpp_env()
 
@@ -449,7 +450,7 @@ def get_pretty_env_info(branch_name):
     if get_platform() == "darwin":
         env_dict.update(npm_env)
         env_info_fmt = env_info_fmt + "\n" + npm_info_fmt
-        
+
     if get_platform() in ("darwin", "linux"):
         env_info_fmt = env_info_fmt + "\n" + cpp_env_info_fmt
 

From 907c3bd28fecea3e80b5a06d97c34e8661514f35 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Sat, 2 Mar 2024 00:58:46 +0000
Subject: [PATCH 44/47] Cleanup ci-cpu-cpp.yml

---
 .github/workflows/ci-cpu-cpp.yml | 26 +++++++++-----------------
 1 file changed, 9 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/ci-cpu-cpp.yml b/.github/workflows/ci-cpu-cpp.yml
index 68de571516..ff7db9cead 100644
--- a/.github/workflows/ci-cpu-cpp.yml
+++ b/.github/workflows/ci-cpu-cpp.yml
@@ -13,18 +13,16 @@ concurrency:
 jobs:
   ci-cpu-cpp:
     runs-on: ${{ matrix.os }}
-    # runs-on: ci-cpp
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-20.04]
-        # os: [ubuntu-20.04, macos-14]
     steps:
-      - name: Setup Python for M1
-        if: matrix.os == 'macos-14'
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
+      # - name: Setup Python for M1
+      #   if: matrix.os == 'macos-14'
+      #   uses: actions/setup-python@v5
+      #   with:
+      #     python-version: '3.10'
       - name: Setup Python for all other OS
         if: matrix.os != 'macos-14'
         uses: actions/setup-python@v5
@@ -40,19 +38,13 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: recursive
-      - name: Install libtorch - macOS
-        if: matrix.os == 'macOS-latest'
-        run: |
-            brew install libtorch
-      - name: Free disk space
-        run: |
-          df -h
+      # - name: Install libtorch - macOS
+      #   if: matrix.os == 'macOS-latest'
+      #   run: |
+      #       brew install libtorch
       - name: Install dependencies
         run: |
           sudo apt update && python ts_scripts/install_dependencies.py --environment=dev --cpp
-      - name: Free disk space
-        run: |
-          df -h
       - name: Install TorchServe
         run: |
           python ts_scripts/install_from_src.py

From d44e968d8416ce76d529f58a46874c7cb58587c4 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Sat, 2 Mar 2024 01:02:04 +0000
Subject: [PATCH 45/47] quieten wget

---
 cpp/build.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/cpp/build.sh b/cpp/build.sh
index 40616c803c..a76c69c5b1 100755
--- a/cpp/build.sh
+++ b/cpp/build.sh
@@ -58,12 +58,12 @@ function install_libtorch() {
     if [ ! -d "$DEPS_DIR/libtorch" ]; then
       if [[ $(uname -m) == 'x86_64' ]]; then
         echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Mac x86_64 ${COLOR_OFF}"
-        wget https://download.pytorch.org/libtorch/cpu/libtorch-macos-x86_64-${TORCH_VERSION}.zip
+        wget -q https://download.pytorch.org/libtorch/cpu/libtorch-macos-x86_64-${TORCH_VERSION}.zip
         unzip -q libtorch-macos-x86_64-${TORCH_VERSION}.zip
         rm libtorch-macos-x86_64-${TORCH_VERSION}.zip
       else
         echo -e "${COLOR_GREEN}[ INFO ] Install libtorch on Mac arm64 ${COLOR_OFF}"
-        wget https://download.pytorch.org/libtorch/cpu/libtorch-macos-arm64-${TORCH_VERSION}.zip
+        wget -q https://download.pytorch.org/libtorch/cpu/libtorch-macos-arm64-${TORCH_VERSION}.zip
         unzip -q libtorch-macos-arm64-${TORCH_VERSION}.zip
         rm libtorch-macos-arm64-${TORCH_VERSION}.zip
       fi
@@ -81,7 +81,7 @@ function install_libtorch() {
       else
         URL=https://download.pytorch.org/libtorch/${CUDA}/libtorch-cxx11-abi-shared-with-deps-${TORCH_VERSION}%2B${CUDA}.zip
       fi
-      wget $URL
+      wget -q $URL
       ZIP_FILE=$(basename "$URL")
       ZIP_FILE="${ZIP_FILE//%2B/+}"
       unzip -q $ZIP_FILE
@@ -98,17 +98,17 @@ function prepare_test_files() {
   local EX_DIR="${TR_DIR}/examples/"
   rsync -a --link-dest=../../test/resources/ ${BASE_DIR}/test/resources/ ${TR_DIR}/
   if [ ! -f "${EX_DIR}/babyllama/babyllama_handler/tokenizer.bin" ]; then
-    wget https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin -O "${EX_DIR}/babyllama/babyllama_handler/tokenizer.bin"
+    wget -q https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin -O "${EX_DIR}/babyllama/babyllama_handler/tokenizer.bin"
   fi
   if [ ! -f "${EX_DIR}/babyllama/babyllama_handler/stories15M.bin" ]; then
-    wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.bin -O "${EX_DIR}/babyllama/babyllama_handler/stories15M.bin"
+    wget -q https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.bin -O "${EX_DIR}/babyllama/babyllama_handler/stories15M.bin"
   fi
   # PT2.2 torch.expport does not support Mac
   if [ "$PLATFORM" = "Linux" ]; then
     if [ ! -f "${EX_DIR}/aot_inductor/llama_handler/stories15M.so" ]; then
       local HANDLER_DIR=${EX_DIR}/aot_inductor/llama_handler/
       if [ ! -f "${HANDLER_DIR}/stories15M.pt" ]; then
-        wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt?download=true -O "${HANDLER_DIR}/stories15M.pt"
+        wget -q https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt?download=true -O "${HANDLER_DIR}/stories15M.pt"
       fi
       local LLAMA_SO_DIR=${BASE_DIR}/third-party/llama2.so/
       PYTHONPATH=${LLAMA_SO_DIR}:${PYTHONPATH} python ${BASE_DIR}/../examples/cpp/aot_inductor/llama2/compile.py --checkpoint ${HANDLER_DIR}/stories15M.pt ${HANDLER_DIR}/stories15M.so

From 9bdfc92a6415e8fa086a522534764401ae94e621 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Sat, 2 Mar 2024 01:15:17 +0000
Subject: [PATCH 46/47] Add build clean section in cpp/readme

---
 cpp/README.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cpp/README.md b/cpp/README.md
index e53dee0afb..cded890b1b 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -26,6 +26,13 @@ cd cpp
 mkdir model_store
 torchserve --ncs --start --model-store model_store
 ```
+
+### Clean the build directory
+To clean the build directory in order to rebuild from scratch simply delete the cpp/_build directory with
+```
+rm -rf cpp/_build
+```
+
 ## Backend
 TorchServe cpp backend can run as a process, which is similar to [TorchServe Python backend](https://github.com/pytorch/serve/tree/master/ts). By default, TorchServe supports torch scripted model in cpp backend. Other platforms such as MxNet, ONNX can be supported through custom handlers following the TorchScript example [src/backends/handler/torch_scripted_handler.hh](https://github.com/pytorch/serve/blob/master/cpp/src/backends/handler/torch_scripted_handler.hh).
 ### Custom Handler

From 9db8ec6000d4e585348cdae4b459e7ff3e24eb83 Mon Sep 17 00:00:00 2001
From: Matthias Reso <13337103+mreso@users.noreply.github.com>
Date: Sat, 2 Mar 2024 04:14:33 +0000
Subject: [PATCH 47/47] Readjust to llama.cpp api

---
 examples/cpp/llamacpp/src/llamacpp_handler.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/cpp/llamacpp/src/llamacpp_handler.cc b/examples/cpp/llamacpp/src/llamacpp_handler.cc
index 52593eb5c6..ce5703a9be 100644
--- a/examples/cpp/llamacpp/src/llamacpp_handler.cc
+++ b/examples/cpp/llamacpp/src/llamacpp_handler.cc
@@ -44,7 +44,7 @@ LlamaCppHandler::LoadModel(
     params.main_gpu = 0;
     params.n_gpu_layers = 35;
 
-    llama_backend_init(false);
+    llama_backend_init();
     ctx_params = llama_context_default_params();
     model_params = llama_model_default_params();
     llamamodel = llama_load_model_from_file(params.model.c_str(), model_params);