From e7c21419d038da77d0c347c9c9227cf971c5fb3b Mon Sep 17 00:00:00 2001 From: Kushal Jain Date: Wed, 24 Apr 2024 15:18:34 +0530 Subject: [PATCH 01/10] improve linux build scripts --- build_jetson.sh | 2 +- build_linux_cuda.sh | 2 +- build_linux_no_cuda.sh | 2 +- .../build_dependencies_jetson_cuda.sh | 41 +++++++++++++---- .../build_dependencies_linux_cuda.sh | 46 +++++++++++++------ .../build_dependencies_linux_no_cuda.sh | 14 +++--- 6 files changed, 72 insertions(+), 35 deletions(-) diff --git a/build_jetson.sh b/build_jetson.sh index 000e8e434..beb3c9da5 100755 --- a/build_jetson.sh +++ b/build_jetson.sh @@ -1,4 +1,4 @@ -apt-get install clang-format +sudo apt-get install clang-format clang-format -style=llvm -dump-config > .clang-format if ! command -v pip &> /dev/null; then # If pip is not available, download and install pip diff --git a/build_linux_cuda.sh b/build_linux_cuda.sh index 52d3ebc49..e8b518f38 100755 --- a/build_linux_cuda.sh +++ b/build_linux_cuda.sh @@ -1,4 +1,4 @@ -apt-get install clang-format +sudo apt-get install clang-format clang-format -style=llvm -dump-config > .clang-format if ! command -v pip &> /dev/null; then # If pip is not available, download and install pip diff --git a/build_linux_no_cuda.sh b/build_linux_no_cuda.sh index 6b9e2d129..3565a4152 100755 --- a/build_linux_no_cuda.sh +++ b/build_linux_no_cuda.sh @@ -1,4 +1,4 @@ -apt-get install clang-format +sudo apt-get install clang-format clang-format -style=llvm -dump-config > .clang-format if ! command -v pip &> /dev/null; then # If pip is not available, download and install pip diff --git a/build_scripts/build_dependencies_jetson_cuda.sh b/build_scripts/build_dependencies_jetson_cuda.sh index 19e10df4e..45954aabf 100644 --- a/build_scripts/build_dependencies_jetson_cuda.sh +++ b/build_scripts/build_dependencies_jetson_cuda.sh @@ -7,7 +7,7 @@ missing_dependencies=() # Check and collect missing dependencies for dependency in "${dependencies[@]}"; do - if ! dpkg -s "$dependency" 2>&1; then + if ! sudo dpkg -s "$dependency" 2>&1; then missing_dependencies+=("$dependency") fi done @@ -15,14 +15,14 @@ done # If there are missing dependencies, install them if [ "${#missing_dependencies[@]}" -gt 0 ]; then echo "Installing missing dependencies..." - apt-get update -qq - apt-get -y install "${missing_dependencies[@]}" + sudo apt-get update -qq + sudo apt-get -y install "${missing_dependencies[@]}" fi # Install Cmake if not present -if ! cmake --version; then +if ! sudo cmake --version; then echo "CMake is not installed. Installing CMake..." - snap install cmake --classic + sudo snap install cmake --classic fi if [ ! -d "/usr/local/cuda/include" ] || [ ! -d "/usr/local/cuda/lib64" ]; then @@ -30,12 +30,33 @@ if [ ! -d "/usr/local/cuda/include" ] || [ ! -d "/usr/local/cuda/lib64" ]; then exit 1 fi -if nvcc --version; then - TARGET_USER="$SUDO_USER" +if sudo nvcc --version; then + userName=$(whoami) + TARGET_USER="$userName" TARGET_HOME=$(eval echo ~$TARGET_USER) - echo 'export VCPKG_FORCE_SYSTEM_BINARIES=1' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc - echo 'export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc - echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + + # Append lines to the target user's ~/.bashrc + if ! grep -qxF 'export VCPKG_FORCE_SYSTEM_BINARIES=1' $TARGET_HOME/.bashrc; then + echo 'export VCPKG_FORCE_SYSTEM_BINARIES=1' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + echo "VCPKG_FORCE_SYSTEM_BINARIES flag added in .bashrc" + else + echo "VCPKG_FORCE_SYSTEM_BINARIES flag already exists in .bashrc" + fi + + if ! grep -qxF 'export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}' $TARGET_HOME/.bashrc; then + echo 'export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + echo "CUDA Binary Path added to .bashrc" + else + echo "CUDA Binary Path already exists in .bashrc" + fi + + if ! grep -qxF 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' $TARGET_HOME/.bashrc; then + echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + echo "CUDA Library Path added to .bashrc" + else + echo "CUDA Library Path already exists in .bashrc" + fi + echo "Appended paths to ~/.bashrc and saved changes." source ~/.bashrc echo "Reloaded ~/.bashrc" diff --git a/build_scripts/build_dependencies_linux_cuda.sh b/build_scripts/build_dependencies_linux_cuda.sh index 2c7a28c89..49ad122c2 100644 --- a/build_scripts/build_dependencies_linux_cuda.sh +++ b/build_scripts/build_dependencies_linux_cuda.sh @@ -11,7 +11,7 @@ missing_dependencies=() # Check and collect missing dependencies for dependency in "${dependencies[@]}"; do - if ! dpkg -s "$dependency" >/dev/null 2>&1; then + if ! sudo dpkg -s "$dependency" >/dev/null 2>&1; then missing_dependencies+=("$dependency") fi done @@ -19,26 +19,26 @@ done # If there are missing dependencies, install them if [ "${#missing_dependencies[@]}" -gt 0 ]; then echo "Installing missing dependencies..." - apt-get update -qq - apt-get -y install "${missing_dependencies[@]}" + sudo apt-get update -qq + sudo apt-get -y install "${missing_dependencies[@]}" fi # Install Meson if not present -if ! meson --version &>/dev/null; then +if ! sudo meson --version &>/dev/null; then echo "meson is not installed. Installing meson..." pip3 install meson fi # Install Cmake if not present -if ! cmake --version &>/dev/null; then +if ! sudo cmake --version &>/dev/null; then echo "CMake is not installed. Installing CMake..." pip3 install cmake --upgrade fi # Install jq if not present -if ! jq --version &>/dev/null; then +if ! sudo jq --version &>/dev/null; then echo "jq is not installed. Installing jq..." - apt install jq + sudo apt install jq fi if [ ! -d "/usr/local/cuda/include" ] || [ ! -d "/usr/local/cuda/lib64" ]; then @@ -46,27 +46,43 @@ if [ ! -d "/usr/local/cuda/include" ] || [ ! -d "/usr/local/cuda/lib64" ]; then exit 1 fi -if ! nvcc --version &>/dev/null; then +if ! sudo nvcc --version &>/dev/null; then userName=$(whoami) cudnn_archives="/home/$userName/Downloads/cudnn-*.tar.xz" for archive in $cudnn_archives; do if [ -e "$archive" ]; then - echo "Extracting $archive..." - tar xf "$archive" -C /home/$userName/Downloads/ + extracted_folder="/home/$userName/Downloads/$(basename "$archive" .tar.xz)" + if [ ! -d "$extracted_folder" ]; then + echo "Extracting $archive..." + tar xf "$archive" -C "/home/$userName/Downloads/" + else + echo "Archive already extracted: $extracted_folder" + fi fi done echo "Copying files..." - cp -r /home/$userName/Downloads/cudnn-*/include/* /usr/local/cuda/include/ - cp -r /home/$userName/Downloads/cudnn-*/lib/* /usr/local/cuda/lib64/ + sudo cp -r /home/$userName/Downloads/cudnn-*/include/* /usr/local/cuda/include/ + sudo cp -r /home/$userName/Downloads/cudnn-*/lib/* /usr/local/cuda/lib64/ - TARGET_USER="$SUDO_USER" + TARGET_USER="$userName" TARGET_HOME=$(eval echo ~$TARGET_USER) # Append lines to the target user's ~/.bashrc - echo 'export PATH=/usr/local/cuda/bin:${PATH}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc - echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:${LD_LIBRARY_PATH}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + if ! grep -qxF 'export PATH=/usr/local/cuda/bin:${PATH}' $TARGET_HOME/.bashrc; then + echo 'export PATH=/usr/local/cuda/bin:${PATH}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + echo "CUDA Binary Path added to .bashrc" + else + echo "CUDA Binary Path already exists in .bashrc" + fi + + if ! grep -qxF 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:${LD_LIBRARY_PATH}' $TARGET_HOME/.bashrc; then + echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:${LD_LIBRARY_PATH}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + echo "CUDA Library Path added to .bashrc" + else + echo "CUDA Library Path already exists in .bashrc" + fi # Reload .bashrc source $TARGET_HOME/.bashrc diff --git a/build_scripts/build_dependencies_linux_no_cuda.sh b/build_scripts/build_dependencies_linux_no_cuda.sh index 49a1d9d69..0fbc87df0 100644 --- a/build_scripts/build_dependencies_linux_no_cuda.sh +++ b/build_scripts/build_dependencies_linux_no_cuda.sh @@ -11,7 +11,7 @@ missing_dependencies=() # Check and collect missing dependencies for dependency in "${dependencies[@]}"; do - if ! dpkg -s "$dependency" >/dev/null 2>&1; then + if ! sudo dpkg -s "$dependency" >/dev/null 2>&1; then missing_dependencies+=("$dependency") fi done @@ -19,26 +19,26 @@ done # If there are missing dependencies, install them if [ "${#missing_dependencies[@]}" -gt 0 ]; then echo "Installing missing dependencies..." - apt-get update -qq - apt-get -y install "${missing_dependencies[@]}" + sudo apt-get update -qq + sudo apt-get -y install "${missing_dependencies[@]}" fi # Install Meson if not present -if ! meson --version &>/dev/null; then +if ! sudo meson --version &>/dev/null; then echo "meson is not installed. Installing meson..." pip3 install meson fi # Install Cmake if not present -if ! cmake --version &>/dev/null; then +if ! sudo cmake --version &>/dev/null; then echo "CMake is not installed. Installing CMake..." pip3 install cmake --upgrade fi # Install jq if not present -if ! jq --version &>/dev/null; then +if ! sudo jq --version &>/dev/null; then echo "jq is not installed. Installing jq..." - apt install jq + sudo apt install jq fi echo "Dependencies verified and installed successfully." From 92383987c37b41647d7664828a2566c2462f2f0a Mon Sep 17 00:00:00 2001 From: Kushal Jain Date: Mon, 29 Apr 2024 16:46:33 +0530 Subject: [PATCH 02/10] update jeston build script --- build_scripts/build_dependencies_jetson_cuda.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build_scripts/build_dependencies_jetson_cuda.sh b/build_scripts/build_dependencies_jetson_cuda.sh index 45954aabf..e76521bf8 100644 --- a/build_scripts/build_dependencies_jetson_cuda.sh +++ b/build_scripts/build_dependencies_jetson_cuda.sh @@ -30,9 +30,9 @@ if [ ! -d "/usr/local/cuda/include" ] || [ ! -d "/usr/local/cuda/lib64" ]; then exit 1 fi -if sudo nvcc --version; then - userName=$(whoami) - TARGET_USER="$userName" +if nvcc --version; then + USER_NAME=$(whoami) + TARGET_USER="$USER_NAME" TARGET_HOME=$(eval echo ~$TARGET_USER) # Append lines to the target user's ~/.bashrc @@ -58,7 +58,7 @@ if sudo nvcc --version; then fi echo "Appended paths to ~/.bashrc and saved changes." - source ~/.bashrc + source $TARGET_USER/.bashrc echo "Reloaded ~/.bashrc" fi From 1e08e8abeeb39b71683b298fa4b817c333ab15d5 Mon Sep 17 00:00:00 2001 From: Kushal Jain Date: Mon, 29 Apr 2024 16:57:13 +0530 Subject: [PATCH 03/10] update build scripts to run .bashrc from home --- build_scripts/build_dependencies_jetson_cuda.sh | 2 +- build_scripts/build_dependencies_linux_cuda.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 build_scripts/build_dependencies_linux_cuda.sh diff --git a/build_scripts/build_dependencies_jetson_cuda.sh b/build_scripts/build_dependencies_jetson_cuda.sh index e76521bf8..ae5032bd2 100644 --- a/build_scripts/build_dependencies_jetson_cuda.sh +++ b/build_scripts/build_dependencies_jetson_cuda.sh @@ -58,7 +58,7 @@ if nvcc --version; then fi echo "Appended paths to ~/.bashrc and saved changes." - source $TARGET_USER/.bashrc + source ~/.bashrc echo "Reloaded ~/.bashrc" fi diff --git a/build_scripts/build_dependencies_linux_cuda.sh b/build_scripts/build_dependencies_linux_cuda.sh old mode 100644 new mode 100755 index 49ad122c2..b68c2bb7c --- a/build_scripts/build_dependencies_linux_cuda.sh +++ b/build_scripts/build_dependencies_linux_cuda.sh @@ -85,7 +85,7 @@ if ! sudo nvcc --version &>/dev/null; then fi # Reload .bashrc - source $TARGET_HOME/.bashrc + source ~/.bashrc echo "Appended line to ~/.bashrc and saved changes." echo "Reloaded ~/.bashrc" From f251767bd215e239a100c94948575ee66ca9129d Mon Sep 17 00:00:00 2001 From: Kushal Jain Date: Mon, 29 Apr 2024 18:25:25 +0530 Subject: [PATCH 04/10] added --build-doc flag in linux scripts --- build_jetson.sh | 6 ++++-- build_linux_cuda.sh | 6 ++++-- build_linux_no_cuda.sh | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/build_jetson.sh b/build_jetson.sh index beb3c9da5..03644e118 100755 --- a/build_jetson.sh +++ b/build_jetson.sh @@ -11,8 +11,10 @@ pre-commit install chmod +x build_scripts/build_dependencies_jetson_cuda.sh ./build_scripts/build_dependencies_jetson_cuda.sh -chmod +x build_documentation.sh -./build_documentation.sh +if [[ $1 == "--build-doc" ]]; then + chmod +x build_documentation.sh + ./build_documentation.sh +fi cd vcpkg ./bootstrap-vcpkg.sh diff --git a/build_linux_cuda.sh b/build_linux_cuda.sh index e8b518f38..cf30c325f 100755 --- a/build_linux_cuda.sh +++ b/build_linux_cuda.sh @@ -11,8 +11,10 @@ pre-commit install chmod +x build_scripts/build_dependencies_linux_cuda.sh ./build_scripts/build_dependencies_linux_cuda.sh -chmod +x build_documentation.sh -./build_documentation.sh +if [[ $1 == "--build-doc" ]]; then + chmod +x build_documentation.sh + ./build_documentation.sh +fi cd vcpkg ./bootstrap-vcpkg.sh diff --git a/build_linux_no_cuda.sh b/build_linux_no_cuda.sh index 3565a4152..56e5709a5 100755 --- a/build_linux_no_cuda.sh +++ b/build_linux_no_cuda.sh @@ -14,8 +14,10 @@ chmod +x build_scripts/build_dependencies_linux_no_cuda.sh chmod +x base/fix-vcpkg-json.sh ./base/fix-vcpkg-json.sh true false false -chmod +x build_documentation.sh -./build_documentation.sh +if [[ $1 == "--build-doc" ]]; then + chmod +x build_documentation.sh + ./build_documentation.sh +fi cd vcpkg ./bootstrap-vcpkg.sh From ef3b751ba24e8d8f8133cfa688038a1e7b889162 Mon Sep 17 00:00:00 2001 From: Kushal Jain Date: Tue, 30 Apr 2024 13:52:55 +0530 Subject: [PATCH 05/10] update windows build scripts --- build_scripts/build_dependencies_windows_cuda.ps1 | 12 ++++++++---- build_windows_cuda.bat | 8 ++++---- build_windows_no_cuda.bat | 6 ++++-- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/build_scripts/build_dependencies_windows_cuda.ps1 b/build_scripts/build_dependencies_windows_cuda.ps1 index ff5218659..0e8ab88d8 100644 --- a/build_scripts/build_dependencies_windows_cuda.ps1 +++ b/build_scripts/build_dependencies_windows_cuda.ps1 @@ -22,10 +22,14 @@ if (-not (Test-Path "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\$cudaVer } else { $userName = $env:UserName $zipFilePath = "C:\Users\$userName\Downloads\cudnn-*.zip" - - Write-Host "Extracting zip file..." - - Expand-Archive -Path $zipFilePath -DestinationPath C:\Users\$userName\Downloads\ -Force + $extractedPath = "C:\Users\$userName\Downloads\cudnn-*\" + + if (-not (Test-Path $extractedPath)) { + Write-Host "Extracting zip file..." + Expand-Archive -Path $zipFilePath -DestinationPath "C:\Users\$userName\Downloads\" -Force + } else { + Write-Host "Already extracted files found." + } Write-Host "Copying files..." Copy-Item -Path "C:\Users\$userName\Downloads\cudnn-*\include\*.h" -Destination "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\$cudaVersion\include\" -Recurse diff --git a/build_windows_cuda.bat b/build_windows_cuda.bat index 0fd221876..ec7022865 100644 --- a/build_windows_cuda.bat +++ b/build_windows_cuda.bat @@ -4,8 +4,10 @@ cd %batdir%/build_scripts powershell -nologo -executionpolicy bypass -File build_dependencies_windows_cuda.ps1 cd .. -@echo off -sh .\build_documentation.sh +IF "%1"=="--build-doc" ( + @echo off + sh .\build_documentation.sh +) @echo off set batdir=%~dp0 @@ -16,8 +18,6 @@ call bootstrap-vcpkg.bat vcpkg.exe integrate install cd .. - - SET VCPKG_ARGS=-DENABLE_CUDA=ON -DENABLE_WINDOWS=ON -DENABLE_LINUX=OFF -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -A x64 ../base mkdir _build diff --git a/build_windows_no_cuda.bat b/build_windows_no_cuda.bat index a4077c902..26e0f95b9 100644 --- a/build_windows_no_cuda.bat +++ b/build_windows_no_cuda.bat @@ -10,8 +10,10 @@ cd %batdir%/base powershell -nologo -executionpolicy bypass -File fix-vcpkg-json.ps1 -removeCUDA cd .. -@echo off -sh .\build_documentation.sh +IF "%1"=="--build-doc" ( + @echo off + sh .\build_documentation.sh +) cd vcpkg call bootstrap-vcpkg.bat From ba20938c6f0ee567e558a3a87c173a182991d56e Mon Sep 17 00:00:00 2001 From: Kushal Jain Date: Wed, 8 May 2024 14:52:12 +0530 Subject: [PATCH 06/10] remove sudo from cloud build scripts --- build_jetson.sh | 34 ++++++++++- build_linux_cuda.sh | 47 ++++++++++++++- build_linux_no_cuda.sh | 2 +- .../build_dependencies_jetson_cuda.sh | 42 ++------------ .../build_dependencies_linux_cuda.sh | 57 ++----------------- .../build_dependencies_linux_no_cuda.sh | 14 ++--- 6 files changed, 98 insertions(+), 98 deletions(-) diff --git a/build_jetson.sh b/build_jetson.sh index 03644e118..ddc05079d 100755 --- a/build_jetson.sh +++ b/build_jetson.sh @@ -9,7 +9,39 @@ pip install pre-commit pre-commit install chmod +x build_scripts/build_dependencies_jetson_cuda.sh -./build_scripts/build_dependencies_jetson_cuda.sh +sudo ./build_scripts/build_dependencies_jetson_cuda.sh + +if nvcc --version; then + USER_NAME=$(whoami) + TARGET_USER="$USER_NAME" + TARGET_HOME=$(eval echo ~$TARGET_USER) + + # Append lines to the target user's ~/.bashrc + if ! grep -qxF 'export VCPKG_FORCE_SYSTEM_BINARIES=1' $TARGET_HOME/.bashrc; then + echo 'export VCPKG_FORCE_SYSTEM_BINARIES=1' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + echo "VCPKG_FORCE_SYSTEM_BINARIES flag added in .bashrc" + else + echo "VCPKG_FORCE_SYSTEM_BINARIES flag already exists in .bashrc" + fi + + if ! grep -qxF 'export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}' $TARGET_HOME/.bashrc; then + echo 'export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + echo "CUDA Binary Path added to .bashrc" + else + echo "CUDA Binary Path already exists in .bashrc" + fi + + if ! grep -qxF 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' $TARGET_HOME/.bashrc; then + echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + echo "CUDA Library Path added to .bashrc" + else + echo "CUDA Library Path already exists in .bashrc" + fi + + echo "Appended paths to ~/.bashrc and saved changes." + source ~/.bashrc + echo "Reloaded ~/.bashrc" +fi if [[ $1 == "--build-doc" ]]; then chmod +x build_documentation.sh diff --git a/build_linux_cuda.sh b/build_linux_cuda.sh index cf30c325f..f37842c01 100755 --- a/build_linux_cuda.sh +++ b/build_linux_cuda.sh @@ -9,7 +9,52 @@ pip install pre-commit pre-commit install chmod +x build_scripts/build_dependencies_linux_cuda.sh -./build_scripts/build_dependencies_linux_cuda.sh +sudo ./build_scripts/build_dependencies_linux_cuda.sh + +if ! sudo nvcc --version &>/dev/null; then + USER_NAME=$(whoami) + cudnn_archives="/home/$USER_NAME/Downloads/cudnn-*.tar.xz" + + for archive in $cudnn_archives; do + if [ -e "$archive" ]; then + extracted_folder="/home/$USER_NAME/Downloads/$(basename "$archive" .tar.xz)" + if [ ! -d "$extracted_folder" ]; then + echo "Extracting $archive..." + tar xf "$archive" -C "/home/$USER_NAME/Downloads/" + else + echo "Archive already extracted: $extracted_folder" + fi + fi + done + + echo "Copying files..." + sudo cp -r /home/$USER_NAME/Downloads/cudnn-*/include/* /usr/local/cuda/include/ + sudo cp -r /home/$USER_NAME/Downloads/cudnn-*/lib/* /usr/local/cuda/lib64/ + + TARGET_USER="$USER_NAME" + TARGET_HOME=$(eval echo ~$TARGET_USER) + + # Append lines to the target user's ~/.bashrc + if ! grep -qxF 'export PATH=/usr/local/cuda/bin:${PATH}' $TARGET_HOME/.bashrc; then + echo 'export PATH=/usr/local/cuda/bin:${PATH}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + echo "CUDA Binary Path added to .bashrc" + else + echo "CUDA Binary Path already exists in .bashrc" + fi + + if ! grep -qxF 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:${LD_LIBRARY_PATH}' $TARGET_HOME/.bashrc; then + echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:${LD_LIBRARY_PATH}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc + echo "CUDA Library Path added to .bashrc" + else + echo "CUDA Library Path already exists in .bashrc" + fi + + # Reload .bashrc + source ~/.bashrc + + echo "Appended line to ~/.bashrc and saved changes." + echo "Reloaded ~/.bashrc" +fi if [[ $1 == "--build-doc" ]]; then chmod +x build_documentation.sh diff --git a/build_linux_no_cuda.sh b/build_linux_no_cuda.sh index 56e5709a5..5da2ce429 100755 --- a/build_linux_no_cuda.sh +++ b/build_linux_no_cuda.sh @@ -9,7 +9,7 @@ pip install pre-commit pre-commit install chmod +x build_scripts/build_dependencies_linux_no_cuda.sh -./build_scripts/build_dependencies_linux_no_cuda.sh +sudo ./build_scripts/build_dependencies_linux_no_cuda.sh chmod +x base/fix-vcpkg-json.sh ./base/fix-vcpkg-json.sh true false false diff --git a/build_scripts/build_dependencies_jetson_cuda.sh b/build_scripts/build_dependencies_jetson_cuda.sh index ae5032bd2..2cdd90949 100644 --- a/build_scripts/build_dependencies_jetson_cuda.sh +++ b/build_scripts/build_dependencies_jetson_cuda.sh @@ -7,7 +7,7 @@ missing_dependencies=() # Check and collect missing dependencies for dependency in "${dependencies[@]}"; do - if ! sudo dpkg -s "$dependency" 2>&1; then + if ! dpkg -s "$dependency" 2>&1; then missing_dependencies+=("$dependency") fi done @@ -15,14 +15,14 @@ done # If there are missing dependencies, install them if [ "${#missing_dependencies[@]}" -gt 0 ]; then echo "Installing missing dependencies..." - sudo apt-get update -qq - sudo apt-get -y install "${missing_dependencies[@]}" + apt-get update -qq + apt-get -y install "${missing_dependencies[@]}" fi # Install Cmake if not present -if ! sudo cmake --version; then +if ! cmake --version; then echo "CMake is not installed. Installing CMake..." - sudo snap install cmake --classic + snap install cmake --classic fi if [ ! -d "/usr/local/cuda/include" ] || [ ! -d "/usr/local/cuda/lib64" ]; then @@ -30,36 +30,4 @@ if [ ! -d "/usr/local/cuda/include" ] || [ ! -d "/usr/local/cuda/lib64" ]; then exit 1 fi -if nvcc --version; then - USER_NAME=$(whoami) - TARGET_USER="$USER_NAME" - TARGET_HOME=$(eval echo ~$TARGET_USER) - - # Append lines to the target user's ~/.bashrc - if ! grep -qxF 'export VCPKG_FORCE_SYSTEM_BINARIES=1' $TARGET_HOME/.bashrc; then - echo 'export VCPKG_FORCE_SYSTEM_BINARIES=1' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc - echo "VCPKG_FORCE_SYSTEM_BINARIES flag added in .bashrc" - else - echo "VCPKG_FORCE_SYSTEM_BINARIES flag already exists in .bashrc" - fi - - if ! grep -qxF 'export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}' $TARGET_HOME/.bashrc; then - echo 'export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc - echo "CUDA Binary Path added to .bashrc" - else - echo "CUDA Binary Path already exists in .bashrc" - fi - - if ! grep -qxF 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' $TARGET_HOME/.bashrc; then - echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc - echo "CUDA Library Path added to .bashrc" - else - echo "CUDA Library Path already exists in .bashrc" - fi - - echo "Appended paths to ~/.bashrc and saved changes." - source ~/.bashrc - echo "Reloaded ~/.bashrc" -fi - echo "Dependencies verified and installed successfully." diff --git a/build_scripts/build_dependencies_linux_cuda.sh b/build_scripts/build_dependencies_linux_cuda.sh index b68c2bb7c..53a026c98 100755 --- a/build_scripts/build_dependencies_linux_cuda.sh +++ b/build_scripts/build_dependencies_linux_cuda.sh @@ -19,26 +19,26 @@ done # If there are missing dependencies, install them if [ "${#missing_dependencies[@]}" -gt 0 ]; then echo "Installing missing dependencies..." - sudo apt-get update -qq - sudo apt-get -y install "${missing_dependencies[@]}" + apt-get update -qq + apt-get -y install "${missing_dependencies[@]}" fi # Install Meson if not present -if ! sudo meson --version &>/dev/null; then +if ! meson --version &>/dev/null; then echo "meson is not installed. Installing meson..." pip3 install meson fi # Install Cmake if not present -if ! sudo cmake --version &>/dev/null; then +if ! cmake --version &>/dev/null; then echo "CMake is not installed. Installing CMake..." pip3 install cmake --upgrade fi # Install jq if not present -if ! sudo jq --version &>/dev/null; then +if ! jq --version &>/dev/null; then echo "jq is not installed. Installing jq..." - sudo apt install jq + apt install jq fi if [ ! -d "/usr/local/cuda/include" ] || [ ! -d "/usr/local/cuda/lib64" ]; then @@ -46,49 +46,4 @@ if [ ! -d "/usr/local/cuda/include" ] || [ ! -d "/usr/local/cuda/lib64" ]; then exit 1 fi -if ! sudo nvcc --version &>/dev/null; then - userName=$(whoami) - cudnn_archives="/home/$userName/Downloads/cudnn-*.tar.xz" - - for archive in $cudnn_archives; do - if [ -e "$archive" ]; then - extracted_folder="/home/$userName/Downloads/$(basename "$archive" .tar.xz)" - if [ ! -d "$extracted_folder" ]; then - echo "Extracting $archive..." - tar xf "$archive" -C "/home/$userName/Downloads/" - else - echo "Archive already extracted: $extracted_folder" - fi - fi - done - - echo "Copying files..." - sudo cp -r /home/$userName/Downloads/cudnn-*/include/* /usr/local/cuda/include/ - sudo cp -r /home/$userName/Downloads/cudnn-*/lib/* /usr/local/cuda/lib64/ - - TARGET_USER="$userName" - TARGET_HOME=$(eval echo ~$TARGET_USER) - - # Append lines to the target user's ~/.bashrc - if ! grep -qxF 'export PATH=/usr/local/cuda/bin:${PATH}' $TARGET_HOME/.bashrc; then - echo 'export PATH=/usr/local/cuda/bin:${PATH}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc - echo "CUDA Binary Path added to .bashrc" - else - echo "CUDA Binary Path already exists in .bashrc" - fi - - if ! grep -qxF 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:${LD_LIBRARY_PATH}' $TARGET_HOME/.bashrc; then - echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:${LD_LIBRARY_PATH}' | sudo -u $TARGET_USER tee -a $TARGET_HOME/.bashrc - echo "CUDA Library Path added to .bashrc" - else - echo "CUDA Library Path already exists in .bashrc" - fi - - # Reload .bashrc - source ~/.bashrc - - echo "Appended line to ~/.bashrc and saved changes." - echo "Reloaded ~/.bashrc" -fi - echo "Dependencies verified and installed successfully." \ No newline at end of file diff --git a/build_scripts/build_dependencies_linux_no_cuda.sh b/build_scripts/build_dependencies_linux_no_cuda.sh index 0fbc87df0..2a90a9680 100644 --- a/build_scripts/build_dependencies_linux_no_cuda.sh +++ b/build_scripts/build_dependencies_linux_no_cuda.sh @@ -11,7 +11,7 @@ missing_dependencies=() # Check and collect missing dependencies for dependency in "${dependencies[@]}"; do - if ! sudo dpkg -s "$dependency" >/dev/null 2>&1; then + if ! dpkg -s "$dependency" >/dev/null 2>&1; then missing_dependencies+=("$dependency") fi done @@ -19,26 +19,26 @@ done # If there are missing dependencies, install them if [ "${#missing_dependencies[@]}" -gt 0 ]; then echo "Installing missing dependencies..." - sudo apt-get update -qq - sudo apt-get -y install "${missing_dependencies[@]}" + apt-get update -qq + apt-get -y install "${missing_dependencies[@]}" fi # Install Meson if not present -if ! sudo meson --version &>/dev/null; then +if ! meson --version &>/dev/null; then echo "meson is not installed. Installing meson..." pip3 install meson fi # Install Cmake if not present -if ! sudo cmake --version &>/dev/null; then +if ! cmake --version &>/dev/null; then echo "CMake is not installed. Installing CMake..." pip3 install cmake --upgrade fi # Install jq if not present -if ! sudo jq --version &>/dev/null; then +if ! jq --version &>/dev/null; then echo "jq is not installed. Installing jq..." - sudo apt install jq + apt install jq fi echo "Dependencies verified and installed successfully." From 6fa45baeba5eea7a78a662e617564f3178908553 Mon Sep 17 00:00:00 2001 From: Kushal Jain Date: Wed, 8 May 2024 15:31:26 +0530 Subject: [PATCH 07/10] update readme with build commands and documentation generation commands --- README.md | 50 +++++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index d83cc69f9..550e4b786 100755 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A pipeline framework for developing video and image processing applications. Sup Learn more about ApraPipes here https://apra-labs.github.io/ApraPipes. ## Build status -Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson Boards (Jetpack 4.4) and Windows (11) x64 Visual Studio 2017 Community. +Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson Boards (Jetpack 4.4) and Windows (11) x64 Visual Studio 2019 Community. |OS|Version|With Cuda|Tests|Status| |--|-------|---------|------|------| |Windows|2019|No|[![Test Results](https://gist.githubusercontent.com/kumaakh/f80af234a4aabedc69af3ee197f66944/raw/badge_Windows.svg)](https://gist.githubusercontent.com/kumaakh/f80af234a4aabedc69af3ee197f66944/raw/badge_Windows.svg)|[![CI-Win-NoCUDA](https://github.com/Apra-Labs/ApraPipes/actions/workflows/CI-Win-NoCUDA.yml/badge.svg)](https://github.com/Apra-Labs/ApraPipes/actions/workflows/CI-Win-NoCUDA.yml)| @@ -40,17 +40,19 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson
Requirements + ### Prerequisites + ### Cuda * Create an account on developer.nvidia.com if you're not already a member. Note : Otherwise the next step will show HTTP 404/403 error. * Windows 10/11 : [Cuda Toolkit 10.2](https://developer.nvidia.com/cuda-10.2-download-archive?target_os=Windows&target_arch=x86_64&target_version=10&target_type=exelocal) or [CUDA Toolkit 11.7](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64). ### Cudnn * Download [Cudnn](https://developer.nvidia.com/rdp/cudnn-archive#a-collapse765-102) and extract files where cuda is installed. Note: Please be aware that this process requires some effort. Here are the necessary steps: - * Download the correct zip file matching your cuda version. _Do not download the exe/installer/deb package._ - * Windows: - * Download [this file](https://developer.nvidia.com/compute/cudnn/secure/8.3.2/local_installers/10.2/cudnn-windows-x86_64-8.3.2.44_cuda10.2-archive.zip). + * Download the correct zip file matching your cuda version. _Do not download the exe/installer/deb package._ + * Windows: + * Download [this file](https://developer.nvidia.com/compute/cudnn/secure/8.3.2/local_installers/10.2/cudnn-windows-x86_64-8.3.2.44_cuda10.2-archive.zip). - ### Prerequisites + ### Visual Studio * Install Visual Studio 2019 Community * Install Desktop development C++ * .NET Desktop development @@ -66,9 +68,9 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson
Build - ### Build Without Cuda Open PowerShell as an administrator and execute the following commands + ### Build Without Cuda If your windows system does not have an NVIDIA GPU use this script ``` build_windows_no_cuda.bat @@ -77,7 +79,10 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson ``` build_windows_cuda.bat ``` - + ### To Build With Documentation + ``` + build_windows_cuda.bat --build-doc + ```
@@ -114,6 +119,8 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson
Requirements + ### Prerequisites + ### Cuda * Create an account on developer.nvidia.com if you're not already a member. Note : Otherwise the next step will show HTTP 404/403 error. * Ubuntu 18.04/20.04: @@ -125,7 +132,8 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson * Linux: * Download [this file](https://developer.nvidia.com/compute/cudnn/secure/8.3.2/local_installers/10.2/cudnn-linux-x86_64-8.3.2.44_cuda10.2-archive.tar.xz) - ### Prerequisites + * Clone with submodules and LFS. + ``` git clone --recursive https://github.com/Apra-Labs/ApraPipes.git ``` @@ -141,11 +149,15 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson ### Build Without Cuda If your windows system does not have an NVIDIA GPU use this script ``` - sudo ./build_linux_no_cuda.sh + ./build_linux_no_cuda.sh ``` ### Build With Cuda ``` - sudo ./build_linux_cuda.sh + ./build_linux_cuda.sh + ``` + ### To Build With Documentation + ``` + ./build_linux_cuda.sh --build-doc ``` Build can take ~2 hours depending on the machine configuration. @@ -200,11 +212,14 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson ``` chmod +x build_jetson.sh ``` - * ApraPipes builds CUDA version on Jerson Boads. + * ApraPipes builds CUDA version on Jetson Boards. ``` - sudo ./build_jetson.sh + ./build_jetson.sh + ``` + * To Build With Documentation + ``` + ./build_jetson.sh --build-doc ``` - Build can take ~12 hours on Jetson Nano. Note: Jetson build can also be done using Ubuntu 18.04 x86_64 Laptop via cross compilation.
@@ -290,13 +305,10 @@ This build will be fairly fast (~10 mins) as entire vcpkg cache comes down with git submodule update --init --recursive ``` ## Update Documentation - After making changes to the documentation located in the /docs/source folder, it's essential to regenerate the documentation by following the provided steps. Once regenerated, commit the new content to ensure the latest documentation is up-to-date. +To update documentation, refer to Documentation Guidelines in the [Contribution-Guidelines](https://github.com/Apra-Labs/ApraPipes/wiki/Contribution-Guidelines). ### To regenerate documentation +Run, ``` -To build docs -apt-install get python-sphinx -pip install sphinx-rtd-theme -cd docs -make html +./build_documentation.sh ``` \ No newline at end of file From 0d4d8e0d3f1503b746407b88bb0201a2cb511072 Mon Sep 17 00:00:00 2001 From: Kushal Jain <155632770+kushaljain-apra@users.noreply.github.com> Date: Fri, 10 May 2024 19:36:40 +0530 Subject: [PATCH 08/10] Update build_dependencies_linux_cuda.sh --- build_scripts/build_dependencies_linux_cuda.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_scripts/build_dependencies_linux_cuda.sh b/build_scripts/build_dependencies_linux_cuda.sh index 53a026c98..f4403333d 100755 --- a/build_scripts/build_dependencies_linux_cuda.sh +++ b/build_scripts/build_dependencies_linux_cuda.sh @@ -11,7 +11,7 @@ missing_dependencies=() # Check and collect missing dependencies for dependency in "${dependencies[@]}"; do - if ! sudo dpkg -s "$dependency" >/dev/null 2>&1; then + if ! dpkg -s "$dependency" >/dev/null 2>&1; then missing_dependencies+=("$dependency") fi done @@ -46,4 +46,4 @@ if [ ! -d "/usr/local/cuda/include" ] || [ ! -d "/usr/local/cuda/lib64" ]; then exit 1 fi -echo "Dependencies verified and installed successfully." \ No newline at end of file +echo "Dependencies verified and installed successfully." From c8de14839bae06015eda4088bd10bdedd83b90b1 Mon Sep 17 00:00:00 2001 From: Kushal Jain <155632770+kushaljain-apra@users.noreply.github.com> Date: Wed, 15 May 2024 12:41:48 +0530 Subject: [PATCH 09/10] Update README.md --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 550e4b786..32e364a68 100755 --- a/README.md +++ b/README.md @@ -41,6 +41,12 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson Requirements ### Prerequisites + + ### Visual Studio + * Install Visual Studio 2019 Community + * Install Desktop development C++ + * .NET Desktop development + * Universal Windows Development Platform ### Cuda * Create an account on developer.nvidia.com if you're not already a member. Note : Otherwise the next step will show HTTP 404/403 error. @@ -51,13 +57,7 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson * Download the correct zip file matching your cuda version. _Do not download the exe/installer/deb package._ * Windows: * Download [this file](https://developer.nvidia.com/compute/cudnn/secure/8.3.2/local_installers/10.2/cudnn-windows-x86_64-8.3.2.44_cuda10.2-archive.zip). - - ### Visual Studio - * Install Visual Studio 2019 Community - * Install Desktop development C++ - * .NET Desktop development - * Universal Windows Development Platform - + * Clone with submodules and LFS. ``` git clone --recursive https://github.com/Apra-Labs/ApraPipes.git @@ -311,4 +311,4 @@ To update documentation, refer to Documentation Guidelines in the [Contribution- Run, ``` ./build_documentation.sh -``` \ No newline at end of file +``` From 0eed1a7d288579b1176fb60861afbbd9cb6bcc52 Mon Sep 17 00:00:00 2001 From: Kushal Jain Date: Mon, 3 Jun 2024 15:56:20 +0530 Subject: [PATCH 10/10] update windows build scripts to build with correct build type --- build_windows_cuda.bat | 4 ++-- build_windows_no_cuda.bat | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build_windows_cuda.bat b/build_windows_cuda.bat index ec7022865..8c98fd8d9 100644 --- a/build_windows_cuda.bat +++ b/build_windows_cuda.bat @@ -23,10 +23,10 @@ SET VCPKG_ARGS=-DENABLE_CUDA=ON -DENABLE_WINDOWS=ON -DENABLE_LINUX=OFF -DCMAKE_T mkdir _build cd _build cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo %VCPKG_ARGS% -cmake --build . +cmake --build . --config RelWithDebInfo cd .. rem goto :EOF mkdir _debugbuild cd _debugbuild cmake -DCMAKE_BUILD_TYPE=Debug %VCPKG_ARGS% -cmake --build . \ No newline at end of file +cmake --build . --config Debug \ No newline at end of file diff --git a/build_windows_no_cuda.bat b/build_windows_no_cuda.bat index 26e0f95b9..abc4ba1d1 100644 --- a/build_windows_no_cuda.bat +++ b/build_windows_no_cuda.bat @@ -23,11 +23,11 @@ cd .. mkdir _buildNoCuda cd _buildNoCuda cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_CUDA=OFF -DENABLE_WINDOWS=ON -DENABLE_LINUX=OFF -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -A x64 ../base -cmake --build . +cmake --build . --config RelWithDebInfo cd .. rem goto :EOF mkdir _debugbuildNoCuda cd _debugbuildNoCuda cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_CUDA=OFF -DENABLE_WINDOWS=ON -DENABLE_LINUX=OFF -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake -A x64 ../base -cmake --build . \ No newline at end of file +cmake --build . --config Debug \ No newline at end of file