From ac41b622f22e305c5f1ae453a727ff2254d8929d Mon Sep 17 00:00:00 2001 From: Olga Vysotska Date: Thu, 7 Nov 2024 16:50:34 +0100 Subject: [PATCH 01/15] add core dependencies installation script + downgrade the pycolmap and numpy for compatibility --- README.md | 14 +++-- install_core_dependencies.sh | 101 +++++++++++++++++++++++++++++++++++ pyproject.toml | 5 +- 3 files changed, 115 insertions(+), 5 deletions(-) create mode 100755 install_core_dependencies.sh diff --git a/README.md b/README.md index 0c1219e..62064d1 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,17 @@ T_w_i = sessions.trajectories[keys[0]] # first pose, from sensor/rig to world :one: Install the core dependencies: -- Python >= 3.9 -- [hloc v1.4](https://github.com/cvg/Hierarchical-Localization) and its dependencies, including [COLMAP 3.8](https://colmap.github.io/install.html) built from source. -- [pyceres](https://github.com/cvg/pyceres) built from source at tag v1.0. +- Python >= 3.9, we recommend to use the virtual environment of your choice. +- Install in order: + * [Ceres 2.1](https://ceres-solver.googlesource.com/ceres-solver/+/refs/tags/2.1.0) + * [COLMAP 3.8](https://colmap.github.io/install.html) built from source. Follow the instructions on the page, but **DO NOT install libceres-dev!!!** We just installed a correct version in the previous step. + * [hloc 1.4](https://github.com/cvg/Hierarchical-Localization) and its dependencies + * [pyceres](https://github.com/cvg/pyceres) built from source at tag v1.0. + + or use the provided script: + ```bash + ./install_core_dependencies.sh + ``` :two: Install the LaMAR libraries and pull the remaining pip dependencies: ```bash diff --git a/install_core_dependencies.sh b/install_core_dependencies.sh new file mode 100755 index 0000000..934bf73 --- /dev/null +++ b/install_core_dependencies.sh @@ -0,0 +1,101 @@ +# Create a directory for all the dependencies +mkdir external +cd external + +####################### +## Installing Ceres-Solver with tag 2.1.0 +## Check webpage https://ceres-solver.googlesource.com/ceres-solver/+/refs/tags/2.1.0 for more details +git clone --no-checkout https://ceres-solver.googlesource.com/ceres-solver ceres_v2.1.0 +cd ceres_v2.1.0 +git checkout f68321e7de8929fbcdb95dd42877531e64f72f66 + +# Installing system-wide dependencies for Ceres +# Commands from http://ceres-solver.org/installation.html +echo "Installing system-wide dependencies cmake build-essential libgoogle-glog-dev libgflags-dev libatlas-base-dev libeigen3-dev libsuitesparse-dev" +sudo apt-get install cmake libgoogle-glog-dev libgflags-dev libatlas-base-dev libeigen3-dev libsuitesparse-dev build-essential + +mkdir build +cd build +cmake .. +make -j4 +sudo make install +cd .. # exit build folder +cd .. # Exiting ceres_v2.1.0 + + +####################### +## Installing COLMAP version 3.8 +git clone https://github.com/colmap/colmap colmap_v3.8 +cd colmap_v3.8 +git checkout 3.8 +git submodule update --init --recursive + +# # Installation instructions from https://colmap.github.io/install.html#build-from-source +echo "Installing system-wide dependencies + ninja-build \ + libboost-program-options-dev \ + libboost-filesystem-dev \ + libboost-graph-dev \ + libboost-system-dev \ + libeigen3-dev \ + libflann-dev \ + libfreeimage-dev \ + libmetis-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + libgmock-dev \ + libsqlite3-dev \ + libglew-dev \ + qtbase5-dev \ + libqt5opengl5-dev \ + libcgal-dev \ " +sudo apt-get install \ + ninja-build \ + libboost-program-options-dev \ + libboost-filesystem-dev \ + libboost-graph-dev \ + libboost-system-dev \ + libeigen3-dev \ + libflann-dev \ + libfreeimage-dev \ + libmetis-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + libgmock-dev \ + libsqlite3-dev \ + libglew-dev \ + qtbase5-dev \ + libqt5opengl5-dev \ + libcgal-dev \ + +mkdir build +cd build +cmake .. -GNinja -DCMAKE_CUDA_ARCHITECTURES=all +ninja +sudo ninja install + +cd .. # exit build folder +cd .. # exit colmap_v3.8 + +####################### +## Installing HLOC version 1.4 +git clone --recursive https://github.com/cvg/Hierarchical-Localization/ hloc_v1.4 +cd hloc_v1.4 +git checkout v1.4 +git submodule update --init --recursive + +# installing +echo "Installing hloc...." +python -m pip install -e . + +cd .. # exit hloc_v1.4 folder + +### Installing pyceres version 1.0 +git clone https://github.com/cvg/pyceres.git pyceres_v1.0 +cd pyceres_v1.0 +git checkout v1.0 +git submodule update --init --recursive + +# installing pyceres +python -m pip install . +cd .. # exit pyceres \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index dc08dc4..55fe9d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,8 @@ dependencies = [ "numpy==1.26.3", "torch>=1.1", "tqdm>=4.36.0", - "pycolmap==0.6.0", + "pycolmap==0.4.0", + "scikit-learn==1.5.2", # is needed for hloc "pyceres @ git+https://github.com/cvg/pyceres.git@v1.0", ] urls = {Repository = "https://github.com/microsoft/lamar-benchmark"} @@ -34,7 +35,7 @@ scantools = [ "pyzbar-upright==0.1.8", "rawpy==0.19.1", "scipy==1.11.4", - "numpy==1.26.4", + "numpy==1.26.3", "pillow==10.3.0", ] dev = [ From f0201d6b37fcd55b1ba82f9b67b9c5bf0059e0d3 Mon Sep 17 00:00:00 2001 From: Olga Vysotska Date: Mon, 11 Nov 2024 10:39:41 +0100 Subject: [PATCH 02/15] stricter version for Python --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 62064d1..dc59e83 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ T_w_i = sessions.trajectories[keys[0]] # first pose, from sensor/rig to world :one: Install the core dependencies: -- Python >= 3.9, we recommend to use the virtual environment of your choice. +- Python 3.9 / 3.10, we recommend to use `venv` virtual environment. - Install in order: * [Ceres 2.1](https://ceres-solver.googlesource.com/ceres-solver/+/refs/tags/2.1.0) * [COLMAP 3.8](https://colmap.github.io/install.html) built from source. Follow the instructions on the page, but **DO NOT install libceres-dev!!!** We just installed a correct version in the previous step. From 66774f1fb49d462e0142c4a506537255077d36b5 Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Tue, 12 Nov 2024 14:18:57 +0100 Subject: [PATCH 03/15] move files --- Dockerfile | 6 +++--- {docker/scripts => scripts}/build_hloc.sh | 0 {docker/scripts => scripts}/build_pcdmeshing.sh | 0 {docker/scripts => scripts}/build_raybender.sh | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename {docker/scripts => scripts}/build_hloc.sh (100%) rename {docker/scripts => scripts}/build_pcdmeshing.sh (100%) rename {docker/scripts => scripts}/build_raybender.sh (100%) diff --git a/Dockerfile b/Dockerfile index 4ef9679..a52a2d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,15 +30,15 @@ RUN apt-get update && \ python3-setuptools # Build raybender. -COPY docker/scripts/build_raybender.sh /tmp/ +COPY scripts/build_raybender.sh /tmp/ RUN bash /tmp/build_raybender.sh && rm /tmp/build_raybender.sh # Build pcdmeshing. -COPY docker/scripts/build_pcdmeshing.sh /tmp/ +COPY scripts/build_pcdmeshing.sh /tmp/ RUN bash /tmp/build_pcdmeshing.sh && rm /tmp/build_pcdmeshing.sh # Build hloc. -COPY docker/scripts/build_hloc.sh /tmp/ +COPY scripts/build_hloc.sh /tmp/ RUN bash /tmp/build_hloc.sh && rm /tmp/build_hloc.sh # diff --git a/docker/scripts/build_hloc.sh b/scripts/build_hloc.sh similarity index 100% rename from docker/scripts/build_hloc.sh rename to scripts/build_hloc.sh diff --git a/docker/scripts/build_pcdmeshing.sh b/scripts/build_pcdmeshing.sh similarity index 100% rename from docker/scripts/build_pcdmeshing.sh rename to scripts/build_pcdmeshing.sh diff --git a/docker/scripts/build_raybender.sh b/scripts/build_raybender.sh similarity index 100% rename from docker/scripts/build_raybender.sh rename to scripts/build_raybender.sh From e4d7946e008ed31d7ec318cfabc0a437d356263d Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Tue, 12 Nov 2024 14:54:03 +0100 Subject: [PATCH 04/15] mv --- .../install_core_dependencies.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename install_core_dependencies.sh => scripts/install_core_dependencies.sh (100%) diff --git a/install_core_dependencies.sh b/scripts/install_core_dependencies.sh similarity index 100% rename from install_core_dependencies.sh rename to scripts/install_core_dependencies.sh From c560c023da406bcc48bc39116e44a5a6929f2cb0 Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Tue, 12 Nov 2024 14:54:40 +0100 Subject: [PATCH 05/15] load env --- scripts/build_hloc.sh | 4 ++-- scripts/build_pcdmeshing.sh | 4 ++-- scripts/build_raybender.sh | 4 ++-- scripts/load_env.sh | 4 ++++ 4 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 scripts/load_env.sh diff --git a/scripts/build_hloc.sh b/scripts/build_hloc.sh index 7c21172..2b3088c 100755 --- a/scripts/build_hloc.sh +++ b/scripts/build_hloc.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -PS4='\033[1;96m$(date +%H:%M:%S)\033[0m ' -set -exo pipefail +root_folder=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/..) +source ${root_folder}/scripts/load_env.sh # Clone hloc. git clone --recursive https://github.com/cvg/Hierarchical-Localization/ hloc --depth=1 diff --git a/scripts/build_pcdmeshing.sh b/scripts/build_pcdmeshing.sh index c2de30d..8965272 100755 --- a/scripts/build_pcdmeshing.sh +++ b/scripts/build_pcdmeshing.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -PS4='\033[1;96m$(date +%H:%M:%S)\033[0m ' -set -exo pipefail +root_folder=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/..) +source ${root_folder}/scripts/load_env.sh sudo apt-get install -y --no-install-recommends --no-install-suggests \ libboost-dev libgmp3-dev libmpfrc++-dev diff --git a/scripts/build_raybender.sh b/scripts/build_raybender.sh index 53697d9..0f60761 100755 --- a/scripts/build_raybender.sh +++ b/scripts/build_raybender.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -PS4='\033[1;96m$(date +%H:%M:%S)\033[0m ' -set -exo pipefail +root_folder=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/..) +source ${root_folder}/scripts/load_env.sh # Clone raybender. git clone --recursive https://github.com/cvg/raybender.git --depth=1 diff --git a/scripts/load_env.sh b/scripts/load_env.sh new file mode 100644 index 0000000..d638c25 --- /dev/null +++ b/scripts/load_env.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +PS4='\033[1;96m$(date "+%Y%m%d %H:%M:%S.%N") $BASH_SOURCE:$LINENO]\033[0m ' +set -euxo pipefail From d0390fa4326c00097e660b3edcd171751c657941 Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Tue, 12 Nov 2024 17:10:08 +0100 Subject: [PATCH 06/15] Split in smaller scripts. --- Dockerfile | 9 ++- README.md | 25 +++--- scripts/build_hloc.sh | 2 +- scripts/install_ceres_solver.sh | 18 +++++ scripts/install_colmap.sh | 28 +++++++ scripts/install_core_dependencies.sh | 109 ++++----------------------- scripts/load_env.sh | 4 +- 7 files changed, 80 insertions(+), 115 deletions(-) create mode 100755 scripts/install_ceres_solver.sh create mode 100755 scripts/install_colmap.sh mode change 100644 => 100755 scripts/load_env.sh diff --git a/Dockerfile b/Dockerfile index a52a2d3..94e57bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,7 +82,7 @@ RUN python3 -m pip install --no-deps \ rawpy==0.19.1 \ scipy==1.11.4 \ numpy==1.26.4 \ - pillow==10.3.0 + pillow==10.3.0 RUN cd lamar && python3 -m pip install -e .[scantools] --no-deps WORKDIR /lamar @@ -123,7 +123,7 @@ RUN apt-get install -y --no-install-recommends --no-install-suggests wget && \ RUN cp -r /ceres_installed/* /usr/local/ # Build pyceres. -RUN git clone --depth 1 --recursive https://github.com/cvg/pyceres +RUN git clone --depth 1 -b v1.0 --recursive https://github.com/cvg/pyceres RUN python3 -m pip install --upgrade pip RUN cd pyceres && \ pip wheel . --no-deps -w dist-wheel -vv && \ @@ -168,13 +168,14 @@ RUN rm -rfv /tmp/* # Note: The dependencies listed in pyproject.toml also include pyceres, already # installed in previous Docker stages. Attempting to compile it in this stage # will lead to failure due to missing necessary development dependencies. -# Therefore, we replicate the dependencies here, excluding pyceres +# Therefore, we replicate the dependencies here, excluding pyceres. RUN python3 -m pip install --no-deps \ h5py==3.10.0 \ numpy==1.26.3 \ torch>=1.1 \ tqdm>=4.36.0 \ - pycolmap==0.6.0 + pycolmap==0.4.0 \ + scikit-learn==1.5.2 RUN cd /lamar && python3 -m pip install -e . --no-deps WORKDIR /lamar diff --git a/README.md b/README.md index dc59e83..fa13897 100644 --- a/README.md +++ b/README.md @@ -66,32 +66,29 @@ T_w_i = sessions.trajectories[keys[0]] # first pose, from sensor/rig to world ## Installation -:one: Install the core dependencies: - -- Python 3.9 / 3.10, we recommend to use `venv` virtual environment. -- Install in order: - * [Ceres 2.1](https://ceres-solver.googlesource.com/ceres-solver/+/refs/tags/2.1.0) - * [COLMAP 3.8](https://colmap.github.io/install.html) built from source. Follow the instructions on the page, but **DO NOT install libceres-dev!!!** We just installed a correct version in the previous step. - * [hloc 1.4](https://github.com/cvg/Hierarchical-Localization) and its dependencies - * [pyceres](https://github.com/cvg/pyceres) built from source at tag v1.0. - - or use the provided script: +:one: Install the **core dependencies** using the provided script: ```bash - ./install_core_dependencies.sh + scripts/install_core_dependencies.sh ``` +Alternatively, you can install them manually in the following order: + * Python 3.9 / 3.10 (we recommend using a `venv` virtual environment). + * [Ceres Solver 2.1](https://ceres-solver.googlesource.com/ceres-solver/+/refs/tags/2.1.0) + * [Colmap 3.8](https://colmap.github.io/install.html) built from source. Note: **Do not install libceres-dev** as it was installed in the previous step. + * [hloc 1.4](https://github.com/cvg/Hierarchical-Localization) and its dependencies + * [pyceres v1.0](https://github.com/cvg/pyceres) built from source. -:two: Install the LaMAR libraries and pull the remaining pip dependencies: +:two: Install LaMAR libraries as editable packages: ```bash python -m pip install -e . ``` -:three: Optional: the processing pipeline additionally relies on heavier dependencies not required for benchmarking: +:three: **Optional**: the processing pipeline additionally relies on heavier dependencies not required for benchmarking: - Pip dependencies: `python -m pip install -e .[scantools]` - [raybender](https://github.com/cvg/raybender) for raytracing - [pcdmeshing](https://github.com/cvg/pcdmeshing) for pointcloud meshing -:four: Optional: if you wish to contribute, install the development tools as well: +:four: **Optional**: if you wish to contribute, install the development tools as well: ```bash python -m pip install -e .[dev] ``` diff --git a/scripts/build_hloc.sh b/scripts/build_hloc.sh index 2b3088c..96233b2 100755 --- a/scripts/build_hloc.sh +++ b/scripts/build_hloc.sh @@ -4,7 +4,7 @@ root_folder=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/..) source ${root_folder}/scripts/load_env.sh # Clone hloc. -git clone --recursive https://github.com/cvg/Hierarchical-Localization/ hloc --depth=1 +git clone --recursive -b v1.4 https://github.com/cvg/Hierarchical-Localization/ hloc --depth=1 cd hloc # Build the wheel. diff --git a/scripts/install_ceres_solver.sh b/scripts/install_ceres_solver.sh new file mode 100755 index 0000000..560438d --- /dev/null +++ b/scripts/install_ceres_solver.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +root_folder=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/..) +source ${root_folder}/scripts/load_env.sh + +apt-get install -y --no-install-recommends --no-install-suggests \ + cmake \ + libgoogle-glog-dev \ + libgflags-dev \ + libatlas-base-dev \ + libeigen3-dev \ + libsuitesparse-dev \ + build-essential + +git clone -b 2.1.0 https://github.com/ceres-solver/ceres-solver.git ceres-solver-v2.1.0 --depth=1 +cd ceres-solver-v2.1.0 +cmake -S . -B build +cmake --build build --target install -- -j$(nproc) diff --git a/scripts/install_colmap.sh b/scripts/install_colmap.sh new file mode 100755 index 0000000..8a82dd1 --- /dev/null +++ b/scripts/install_colmap.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +root_folder=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/..) +source ${root_folder}/scripts/load_env.sh + +apt-get install -y --no-install-recommends --no-install-suggests \ + libboost-program-options-dev \ + libboost-filesystem-dev \ + libboost-graph-dev \ + libboost-system-dev \ + libeigen3-dev \ + libflann-dev \ + libfreeimage-dev \ + libmetis-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + libgmock-dev \ + libsqlite3-dev \ + libglew-dev \ + qtbase5-dev \ + libqt5opengl5-dev \ + libcgal-dev + +# Installing COLMAP version 3.8 +git clone --recursive -b 3.8 https://github.com/colmap/colmap colmap_v3.8 --depth=1 +cd colmap_v3.8 +cmake -S . -B build -DCMAKE_CUDA_ARCHITECTURES=all +cmake --build build --target install -- -j$(nproc) diff --git a/scripts/install_core_dependencies.sh b/scripts/install_core_dependencies.sh index 934bf73..23b582c 100755 --- a/scripts/install_core_dependencies.sh +++ b/scripts/install_core_dependencies.sh @@ -1,101 +1,20 @@ -# Create a directory for all the dependencies -mkdir external -cd external +root_folder=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/..) +source ${root_folder}/scripts/load_env.sh -####################### -## Installing Ceres-Solver with tag 2.1.0 -## Check webpage https://ceres-solver.googlesource.com/ceres-solver/+/refs/tags/2.1.0 for more details -git clone --no-checkout https://ceres-solver.googlesource.com/ceres-solver ceres_v2.1.0 -cd ceres_v2.1.0 -git checkout f68321e7de8929fbcdb95dd42877531e64f72f66 +# Comment this line out if you want to use it inside docker. +# apt-get update && apt-get install -y --no-install-recommends --no-install-suggests git python3 python3-dev python3-pip python-is-python3 sudo -# Installing system-wide dependencies for Ceres -# Commands from http://ceres-solver.org/installation.html -echo "Installing system-wide dependencies cmake build-essential libgoogle-glog-dev libgflags-dev libatlas-base-dev libeigen3-dev libsuitesparse-dev" -sudo apt-get install cmake libgoogle-glog-dev libgflags-dev libatlas-base-dev libeigen3-dev libsuitesparse-dev build-essential +# Create external folder. +mkdir ${root_folder}/external && cd ${root_folder}/external -mkdir build -cd build -cmake .. -make -j4 -sudo make install -cd .. # exit build folder -cd .. # Exiting ceres_v2.1.0 +# Ceres Solver. +sudo ${root_folder}/scripts/install_ceres_solver.sh +# Colmap. +sudo ${root_folder}/scripts/install_colmap.sh -####################### -## Installing COLMAP version 3.8 -git clone https://github.com/colmap/colmap colmap_v3.8 -cd colmap_v3.8 -git checkout 3.8 -git submodule update --init --recursive +# HLoc. +python3 -m pip install git+https://github.com/cvg/Hierarchical-Localization.git@v1.4 -# # Installation instructions from https://colmap.github.io/install.html#build-from-source -echo "Installing system-wide dependencies - ninja-build \ - libboost-program-options-dev \ - libboost-filesystem-dev \ - libboost-graph-dev \ - libboost-system-dev \ - libeigen3-dev \ - libflann-dev \ - libfreeimage-dev \ - libmetis-dev \ - libgoogle-glog-dev \ - libgtest-dev \ - libgmock-dev \ - libsqlite3-dev \ - libglew-dev \ - qtbase5-dev \ - libqt5opengl5-dev \ - libcgal-dev \ " -sudo apt-get install \ - ninja-build \ - libboost-program-options-dev \ - libboost-filesystem-dev \ - libboost-graph-dev \ - libboost-system-dev \ - libeigen3-dev \ - libflann-dev \ - libfreeimage-dev \ - libmetis-dev \ - libgoogle-glog-dev \ - libgtest-dev \ - libgmock-dev \ - libsqlite3-dev \ - libglew-dev \ - qtbase5-dev \ - libqt5opengl5-dev \ - libcgal-dev \ - -mkdir build -cd build -cmake .. -GNinja -DCMAKE_CUDA_ARCHITECTURES=all -ninja -sudo ninja install - -cd .. # exit build folder -cd .. # exit colmap_v3.8 - -####################### -## Installing HLOC version 1.4 -git clone --recursive https://github.com/cvg/Hierarchical-Localization/ hloc_v1.4 -cd hloc_v1.4 -git checkout v1.4 -git submodule update --init --recursive - -# installing -echo "Installing hloc...." -python -m pip install -e . - -cd .. # exit hloc_v1.4 folder - -### Installing pyceres version 1.0 -git clone https://github.com/cvg/pyceres.git pyceres_v1.0 -cd pyceres_v1.0 -git checkout v1.0 -git submodule update --init --recursive - -# installing pyceres -python -m pip install . -cd .. # exit pyceres \ No newline at end of file +# Pyceres. +python3 -m pip install git+https://github.com/cvg/pyceres.git@v1.0 diff --git a/scripts/load_env.sh b/scripts/load_env.sh old mode 100644 new mode 100755 index d638c25..4d78858 --- a/scripts/load_env.sh +++ b/scripts/load_env.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash -PS4='\033[1;96m$(date "+%Y%m%d %H:%M:%S.%N") $BASH_SOURCE:$LINENO]\033[0m ' +# PS4='\033[1;96m$(date +%H:%M:%S)\033[0m ' + +PS4='\033[0;32m$(date "+%Y%m%d %H:%M:%S.%N") $BASH_SOURCE:$LINENO]\033[0m ' set -euxo pipefail From 31cfa26bfe185ceb2d012180777ac45cad0cac24 Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Tue, 12 Nov 2024 21:49:47 +0100 Subject: [PATCH 07/15] update pyceres to v2.3 (v1.0 depended on Colmap and failed to install in the Docker image). --- Dockerfile | 2 +- scripts/install_core_dependencies.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 94e57bf..7d168b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,7 +123,7 @@ RUN apt-get install -y --no-install-recommends --no-install-suggests wget && \ RUN cp -r /ceres_installed/* /usr/local/ # Build pyceres. -RUN git clone --depth 1 -b v1.0 --recursive https://github.com/cvg/pyceres +RUN git clone --depth 1 -b v2.3 --recursive https://github.com/cvg/pyceres RUN python3 -m pip install --upgrade pip RUN cd pyceres && \ pip wheel . --no-deps -w dist-wheel -vv && \ diff --git a/scripts/install_core_dependencies.sh b/scripts/install_core_dependencies.sh index 23b582c..f98fc96 100755 --- a/scripts/install_core_dependencies.sh +++ b/scripts/install_core_dependencies.sh @@ -17,4 +17,4 @@ sudo ${root_folder}/scripts/install_colmap.sh python3 -m pip install git+https://github.com/cvg/Hierarchical-Localization.git@v1.4 # Pyceres. -python3 -m pip install git+https://github.com/cvg/pyceres.git@v1.0 +python3 -m pip install git+https://github.com/cvg/pyceres.git@v2.3 From 9383d64db2b549daae78e6b8d1888f76c984b43f Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Thu, 14 Nov 2024 16:41:10 +0100 Subject: [PATCH 08/15] Update scripts/install_core_dependencies.sh --- scripts/install_core_dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_core_dependencies.sh b/scripts/install_core_dependencies.sh index f98fc96..23b582c 100755 --- a/scripts/install_core_dependencies.sh +++ b/scripts/install_core_dependencies.sh @@ -17,4 +17,4 @@ sudo ${root_folder}/scripts/install_colmap.sh python3 -m pip install git+https://github.com/cvg/Hierarchical-Localization.git@v1.4 # Pyceres. -python3 -m pip install git+https://github.com/cvg/pyceres.git@v2.3 +python3 -m pip install git+https://github.com/cvg/pyceres.git@v1.0 From 1674e63578b65668fd9280ef7bc5d5ee675cabcf Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Fri, 15 Nov 2024 14:53:42 +0100 Subject: [PATCH 09/15] update Dockerfile with the new scripts (notice we don't split in builder/runtime, so docker images will be larger) --- Dockerfile | 49 +++++++++++++++---------------------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7d168b2..1521807 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,8 +64,7 @@ RUN rm -rfv /tmp/* # Install pcdmeshing. COPY --from=builder /pcdmeshing/dist-wheel /tmp/dist-wheel -RUN sudo apt-get install -y --no-install-recommends --no-install-suggests \ - libmpfrc++-dev +RUN apt-get install -y --no-install-recommends --no-install-suggests libmpfrc++-dev RUN cd /tmp && whl_path=$(cat dist-wheel/whl_path.txt) && python3 -m pip install $whl_path RUN rm -rfv /tmp/* @@ -90,41 +89,23 @@ WORKDIR /lamar # # pyceres-builder stage. # -FROM mcr.microsoft.com/mirror/docker/library/ubuntu:${UBUNTU_VERSION} AS pyceres-builder +FROM common AS pyceres-builder -# Prepare and empty machine for building. -RUN apt-get update && \ - apt-get install -y --no-install-recommends --no-install-suggests \ - git \ - cmake \ - ninja-build \ - build-essential \ - libeigen3-dev \ - libgoogle-glog-dev \ - libgflags-dev \ - libgtest-dev \ - libatlas-base-dev \ - libsuitesparse-dev \ - python-is-python3 \ - python3-minimal \ - python3-pip \ - python3-dev \ - python3-setuptools +# Copy scripts +COPY scripts/* /tmp/ + +# Install Ceres Solver. +COPY scripts/install_ceres_solver.sh /tmp/ +RUN bash /tmp/install_ceres_solver.sh -# Install Ceres. -RUN apt-get install -y --no-install-recommends --no-install-suggests wget && \ - wget "http://ceres-solver.org/ceres-solver-2.1.0.tar.gz" && \ - tar zxf ceres-solver-2.1.0.tar.gz && \ - mkdir ceres-build && \ - cd ceres-build && \ - cmake ../ceres-solver-2.1.0 -GNinja \ - -DCMAKE_INSTALL_PREFIX=/ceres_installed && \ - ninja install -RUN cp -r /ceres_installed/* /usr/local/ +# Install Colmap. +COPY scripts/install_colmap.sh /tmp/ +RUN bash /tmp/install_colmap.sh # Build pyceres. -RUN git clone --depth 1 -b v2.3 --recursive https://github.com/cvg/pyceres +RUN git clone --depth 1 -b v1.0 --recursive https://github.com/cvg/pyceres RUN python3 -m pip install --upgrade pip +RUN apt-get install -y --no-install-recommends --no-install-suggests python3-dev RUN cd pyceres && \ pip wheel . --no-deps -w dist-wheel -vv && \ whl_path=$(find dist-wheel/ -name "*.whl") && \ @@ -146,8 +127,8 @@ RUN apt-get update && \ python3-minimal \ python3-pip -# Copy installed library in the builder stage. -COPY --from=pyceres-builder /ceres_installed/ /usr/local/ +# Copy installed libraries in the builder stage. +COPY --from=pyceres-builder /usr/local/ /usr/local/ # Install pyceres. COPY --from=pyceres-builder /pyceres/dist-wheel /tmp/dist-wheel From 5abb54f2ee43cbb488e8833d2aee4c63a4dfbb12 Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Fri, 15 Nov 2024 14:54:21 +0100 Subject: [PATCH 10/15] Usage of docker images --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index fa13897..40ec005 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,32 @@ docker pull ghcr.io/microsoft/lamar-benchmark/scantools:latest docker pull ghcr.io/microsoft/lamar-benchmark/lamar:latest ``` +### Usage of docker images + +To use the `lamar` Docker image, you can follow these steps: + +1. **Set the `DATA_DIR` and `DOCKER_RUN` environment variables**: + +```bash +export DATA_DIR=/path/to/data +export DOCKER_RUN="docker run -it --rm --init -u $(id -u):$(id -g) -v ${DATA_DIR}:${DATA_DIR} lamar:lamar " +``` + +**Note**: replace `lamar:lamar` with `ghcr.io/microsoft/lamar-benchmark/lamar:latest` +if you want to use the image from the GitHub Docker Registry. + +2. **Run the desired command inside the Docker container, for example**: + +```bash +$DOCKER_RUN ls $DATA_DIR +$DOCKER_RUN python pipelines/pipeline_navvis_rig.py --help +``` + +The `DOCKER_RUN` variable is a prefix to the command you would run if the code +were installed locally. This ensures the command runs inside the Docker +container. The `DATA_DIR` will be mounted as a volume representing the same +folder on the local machine, and the user/group will match the local environment +to avoid having the output as root. ## Benchmark From 939891e2603b8e52129647ef6721392b4942a33a Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Fri, 15 Nov 2024 15:26:40 +0100 Subject: [PATCH 11/15] update README, docker usage --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 40ec005..464337d 100644 --- a/README.md +++ b/README.md @@ -129,11 +129,11 @@ To use the `lamar` Docker image, you can follow these steps: ```bash export DATA_DIR=/path/to/data -export DOCKER_RUN="docker run -it --rm --init -u $(id -u):$(id -g) -v ${DATA_DIR}:${DATA_DIR} lamar:lamar " +export DOCKER_RUN="docker run -it --rm --init -u $(id -u):$(id -g) -v ${DATA_DIR}:${DATA_DIR} ghcr.io/microsoft/lamar-benchmark/lamar:latest " ``` -**Note**: replace `lamar:lamar` with `ghcr.io/microsoft/lamar-benchmark/lamar:latest` -if you want to use the image from the GitHub Docker Registry. +**Note**: replace `ghcr.io/microsoft/lamar-benchmark/lamar:latest` with +`lamar:lamar` if you want to use the image you built locally. 2. **Run the desired command inside the Docker container, for example**: From fdfc0b399bde0b0f53a04623ac72838597a2e5bb Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Mon, 18 Nov 2024 09:40:31 +0100 Subject: [PATCH 12/15] simplify hloc installation --- Dockerfile | 8 +------- scripts/build_hloc.sh | 13 ------------- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100755 scripts/build_hloc.sh diff --git a/Dockerfile b/Dockerfile index 1521807..ff7d994 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,10 +37,6 @@ RUN bash /tmp/build_raybender.sh && rm /tmp/build_raybender.sh COPY scripts/build_pcdmeshing.sh /tmp/ RUN bash /tmp/build_pcdmeshing.sh && rm /tmp/build_pcdmeshing.sh -# Build hloc. -COPY scripts/build_hloc.sh /tmp/ -RUN bash /tmp/build_hloc.sh && rm /tmp/build_hloc.sh - # # Scantools stage. # @@ -142,9 +138,7 @@ RUN rm -rfv /tmp/* FROM pyceres as lamar # Install hloc. -COPY --from=builder /hloc/dist-wheel /tmp/dist-wheel -RUN cd /tmp && whl_path=$(cat dist-wheel/whl_path.txt) && python3 -m pip install $whl_path -RUN rm -rfv /tmp/* +RUN python3 -m pip install git+https://github.com/cvg/Hierarchical-Localization.git@v1.4 # Note: The dependencies listed in pyproject.toml also include pyceres, already # installed in previous Docker stages. Attempting to compile it in this stage diff --git a/scripts/build_hloc.sh b/scripts/build_hloc.sh deleted file mode 100755 index 96233b2..0000000 --- a/scripts/build_hloc.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -root_folder=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/..) -source ${root_folder}/scripts/load_env.sh - -# Clone hloc. -git clone --recursive -b v1.4 https://github.com/cvg/Hierarchical-Localization/ hloc --depth=1 -cd hloc - -# Build the wheel. -pip wheel --no-deps -w dist-wheel . -whl_path=$(find dist-wheel/ -name "*.whl") -echo $whl_path >dist-wheel/whl_path.txt From 855c45b1087d4d0bbe42a37bad4d495aebd48a56 Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Thu, 21 Nov 2024 17:45:16 +0100 Subject: [PATCH 13/15] hloc, using `-e` option in pip install --- Dockerfile | 3 ++- scripts/install_core_dependencies.sh | 4 +++- scripts/install_hloc.sh | 8 ++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100755 scripts/install_hloc.sh diff --git a/Dockerfile b/Dockerfile index ff7d994..ead5864 100644 --- a/Dockerfile +++ b/Dockerfile @@ -138,7 +138,8 @@ RUN rm -rfv /tmp/* FROM pyceres as lamar # Install hloc. -RUN python3 -m pip install git+https://github.com/cvg/Hierarchical-Localization.git@v1.4 +COPY scripts/install_hloc.sh /tmp/ +RUN bash /tmp/install_hloc.sh # Note: The dependencies listed in pyproject.toml also include pyceres, already # installed in previous Docker stages. Attempting to compile it in this stage diff --git a/scripts/install_core_dependencies.sh b/scripts/install_core_dependencies.sh index 23b582c..a0d563c 100755 --- a/scripts/install_core_dependencies.sh +++ b/scripts/install_core_dependencies.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + root_folder=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/..) source ${root_folder}/scripts/load_env.sh @@ -14,7 +16,7 @@ sudo ${root_folder}/scripts/install_ceres_solver.sh sudo ${root_folder}/scripts/install_colmap.sh # HLoc. -python3 -m pip install git+https://github.com/cvg/Hierarchical-Localization.git@v1.4 +${root_folder}/scripts/install_hloc.sh # Pyceres. python3 -m pip install git+https://github.com/cvg/pyceres.git@v1.0 diff --git a/scripts/install_hloc.sh b/scripts/install_hloc.sh new file mode 100755 index 0000000..65399a3 --- /dev/null +++ b/scripts/install_hloc.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +root_folder=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/..) +source ${root_folder}/scripts/load_env.sh + +git clone --recursive -b v1.4 https://github.com/cvg/Hierarchical-Localization/ hloc --depth=1 +cd hloc +python -m pip install -e . From c6ef7f516d638b9e6ef0881561f5fde48e375f6e Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Thu, 21 Nov 2024 21:50:45 +0100 Subject: [PATCH 14/15] clean-up --- scripts/install_core_dependencies.sh | 2 +- scripts/load_env.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/install_core_dependencies.sh b/scripts/install_core_dependencies.sh index a0d563c..cb54ba4 100755 --- a/scripts/install_core_dependencies.sh +++ b/scripts/install_core_dependencies.sh @@ -3,7 +3,7 @@ root_folder=$(realpath $(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/..) source ${root_folder}/scripts/load_env.sh -# Comment this line out if you want to use it inside docker. +# Uncomment the following line if you want to use this script inside Docker. # apt-get update && apt-get install -y --no-install-recommends --no-install-suggests git python3 python3-dev python3-pip python-is-python3 sudo # Create external folder. diff --git a/scripts/load_env.sh b/scripts/load_env.sh index 4d78858..76c8ec7 100755 --- a/scripts/load_env.sh +++ b/scripts/load_env.sh @@ -1,6 +1,4 @@ #!/usr/bin/env bash -# PS4='\033[1;96m$(date +%H:%M:%S)\033[0m ' - PS4='\033[0;32m$(date "+%Y%m%d %H:%M:%S.%N") $BASH_SOURCE:$LINENO]\033[0m ' set -euxo pipefail From e72bd4b52b3cb91ce1de44729387b70f5c10f79f Mon Sep 17 00:00:00 2001 From: Pablo Speciale Date: Tue, 3 Dec 2024 15:58:57 +0100 Subject: [PATCH 15/15] CR: ubuntu 22.04 --- README.md | 3 +-- scripts/install_core_dependencies.sh | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 464337d..b955883 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ T_w_i = sessions.trajectories[keys[0]] # first pose, from sensor/rig to world ## Installation -:one: Install the **core dependencies** using the provided script: +:one: Install the **core dependencies** using the provided script, tested on Ubuntu 22.04: ```bash scripts/install_core_dependencies.sh ``` @@ -75,7 +75,6 @@ Alternatively, you can install them manually in the following order: * [Ceres Solver 2.1](https://ceres-solver.googlesource.com/ceres-solver/+/refs/tags/2.1.0) * [Colmap 3.8](https://colmap.github.io/install.html) built from source. Note: **Do not install libceres-dev** as it was installed in the previous step. * [hloc 1.4](https://github.com/cvg/Hierarchical-Localization) and its dependencies - * [pyceres v1.0](https://github.com/cvg/pyceres) built from source. :two: Install LaMAR libraries as editable packages: ```bash diff --git a/scripts/install_core_dependencies.sh b/scripts/install_core_dependencies.sh index cb54ba4..435f031 100755 --- a/scripts/install_core_dependencies.sh +++ b/scripts/install_core_dependencies.sh @@ -17,6 +17,3 @@ sudo ${root_folder}/scripts/install_colmap.sh # HLoc. ${root_folder}/scripts/install_hloc.sh - -# Pyceres. -python3 -m pip install git+https://github.com/cvg/pyceres.git@v1.0