Merge branch 'main' into upgrade-torch

.dockerignore

@@ -0,0 +1,14 @@
+.hydra
+.vscode
+htmlcov
+
+# Caches
+**/__pycache__
+.pytest_cache
+
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info
+*.egg

.github/workflows/cicd.yaml

@@ -25,7 +25,7 @@ jobs:
       uses: actions/checkout@v3
 
     - name: Build docker image
-      run: docker build -t myria3d .
+      run: docker build --build-arg http_proxy=${{ secrets.PROXY_URL }} --build-arg https_proxy=${{ secrets.PROXY_URL }} -t myria3d .
 
     - name: Run pytest
       run: >
@@ -36,7 +36,7 @@ jobs:
         pytest -rA -v
         --ignore=actions-runner
 
-    # IMPORTANT: Always run images with --ipc=host and --shm-size=2gb (at least) to enable 
+    # IMPORTANT: Always run images with --ipc=host and --shm-size=2gb (at least) to enable
     # sufficient shared memory when predicting on large files.
     - name: Example inference run via Docker with default config and checkpoint
       run: >
@@ -77,12 +77,12 @@ jobs:
       run: docker run myria3d python -m flake8
 
     # Everything ran so we tag the valid docker image to keep it
-    # This happens for push events, which are in particular 
+    # This happens for push events, which are in particular
     # triggered when a pull request is merged.
     - name: Tag the docker image with branch name
       if: github.event_name == 'push'
       run: |
-        docker tag myria3d:latest myria3d:${{github.ref_name}} 
+        docker tag myria3d:latest myria3d:${{github.ref_name}}
         docker run myria3d:${{github.ref_name}} bash # Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune
         # docker save myria3d:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${github.ref_name}.tar # Save the docker image as myria3d_${github.ref_name}.tar
 
@@ -101,7 +101,7 @@ jobs:
 
     - name: push main docker on nexus (tagged with a date)
       # we push on nexus an image from the main branch when it has been updated (push or accepted pull request)
-      if: ((github.ref_name == 'main') && (github.event_name == 'push')) 
+      if: ((github.ref_name == 'main') && (github.event_name == 'push'))
       run: |
         docker tag myria3d $nexus_server/lidar_hd/myria3d:${{steps.tag.outputs.version}}-${{steps.tag.outputs.date}}
         docker login $nexus_server --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }}

Dockerfile

@@ -1,49 +1,33 @@
-FROM nvidia/cuda:11.3.1-base-ubuntu20.04
-
-# set the IGN proxy, otherwise apt-get and other applications don't work
-# Should be commented out outside of IGN
-ENV http_proxy 'http://192.168.4.9:3128/'
-ENV https_proxy 'http://192.168.4.9:3128/'
-
-# Remove any third-party apt sources to avoid issues with expiring keys.
-RUN rm -f /etc/apt/sources.list.d/*.list
-
-# Install some basic utilities
-RUN apt-get update && apt-get install -y \
-        nano \
-        curl \
-        ca-certificates \
-        sudo \
-        git \
-        bzip2 \
-        libx11-6 \
-        && rm -rf /var/lib/apt/lists/*
-
-# Create a working directory
-RUN mkdir /app
-
-# Set up the Conda environment and make python accessible via PATH.
-ENV CONDA_AUTO_UPDATE_CONDA=false
-ENV PATH=/miniconda:/miniconda/bin:$PATH 
-COPY environment.yml /app/environment.yml
-RUN curl -sLo /miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh \
-        && chmod +x /miniconda.sh \
-        && /miniconda.sh -b -p /miniconda \
-        && rm /miniconda.sh \
-        && /miniconda/bin/conda env update -n base -f /app/environment.yml \
-        && rm /app/environment.yml \
-        && /miniconda/bin/conda clean -ya
+FROM mambaorg/micromamba:focal-cuda-11.3.1
+# focal is Ubuntu 20.04
 
+WORKDIR /app
+
+COPY environment.yml environment.yml
+
+# Switching to root does not seem necessary in the general case, but the github ci/cd process
+# does not seem to work without (rresults in a permission error when running pip packages
+# installation similar to https://github.com/mamba-org/micromamba-docker/issues/356)
+USER root
+
+RUN micromamba env create -f /app/environment.yml
+
+ENV PATH=$PATH:/opt/conda/envs/myria3d/bin/
 # Need to export this for torch_geometric to find where cuda is.
 # See https://github.com/pyg-team/pytorch_geometric/issues/2040#issuecomment-766610625
-ENV LD_LIBRARY_PATH="/miniconda/lib/:$LD_LIBRARY_PATH"
+ENV LD_LIBRARY_PATH="/opt/conda/envs/myria3d/lib/:$LD_LIBRARY_PATH"
 
-# Check succes of environment creation.
+# Check success of environment creation.
 RUN python -c "import torch_geometric;"
 
-# Copy the repository content in /app 
-WORKDIR /app
+# use chown to prevent permission issues
 COPY . .
 
-# Set the default command to bash for image inspection.
-CMD ["bash"]
+# locate proj
+ENV PROJ_LIB=/opt/conda/envs/myria3d/share/proj/
+
+# Check that myria3d can run
+RUN python run.py task.task_name=predict --help
+
+# # Set the default command to bash for image inspection.
+# CMD ["bash"]

environment.yml

@@ -8,22 +8,40 @@ channels:
   - comet_ml
   - conda-forge
   - anaconda
+  - pytorch
+  - comet_ml
+  - pyg
 dependencies:
   - python=3.9.*
   - pip
-  # --------- Deep Learning --------- #
-  - pytorch=2.1
-  - pytorch-cuda=11.8
-  - torchvision=0.16
-  - lightning=2.0
-  - torchmetrics=0.11
-  - pyg=2.4
   # --------- data formats --------- #
   - h5py
+  - numpy
+  # --------- Deep Learning --------- #
+  # cudatoolkit to specify the cuda driver in the conda env
+  - conda-forge::cudatoolkit=11.3.1 # single equal sign there, not a typo
+  - pytorch::pytorch==1.11.0
+  - pytorch::torchvision==0.12.0
+  - conda-forge::pytorch-lightning==1.5.9
+  - conda-forge::torchmetrics==0.7.*
+  - comet_ml::comet_ml==3.31.*
+  - conda-forge::urllib3<2  # To solve for https://github.com/GeneralMills/pytrends/issues/591
+  - pyg::pytorch-cluster
+  - pyg::pytorch-scatter
+  - pyg::pytorch-sparse
+  - pyg::pyg==2.1.0
+    # Nota: if libcusparse.so.11. errors occur, run
+    # export LD_LIBRARY_PATH="/home/${USER}/miniconda/envs/lib:$LD_LIBRARY_PATH"
+    # ou
+    # export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/lib:$LD_LIBRARY_PATH"
+    # see https://github.com/pyg-team/pytorch_geometric/issues/2040#issuecomment-766610625
   # --------- geo --------- #
-  - pygeos
-  - shapely
+  - pdal
   - python-pdal
+  - pyproj
+    # --------- Visualization --------- #
+  - pandas
+  - matplotlib
   # --------- loggers --------- #
   - comet_ml=3.35
     # --------- Visualization --------- #
@@ -44,28 +62,17 @@ dependencies:
   - sh # for running bash commands in some tests
   - seaborn # used in some callbacks
   - pudb # debugger
+    # # --------- Documentation --------- #
+  - sphinx==4.5.*
+  - recommonmark==0.7.*
+  - sphinx_rtd_theme==1.0.*
+  - docutils==0.17
+  - rstcheck==3.3.*  # RST Linter
   - pip:
-      # Extra index may need to be on first line 
-      - urllib3<2  # To solve for https://github.com/GeneralMills/pytrends/issues/591
-      # Wheels for torch-geometric optionnal dependencies
-      - https://data.pyg.org/whl/torch-2.1.0%2Bcu118/torch_cluster-1.6.3%2Bpt21cu118-cp39-cp39-linux_x86_64.whl
-      - https://data.pyg.org/whl/torch-2.1.0%2Bcu118/torch_scatter-2.1.2%2Bpt21cu118-cp39-cp39-linux_x86_64.whl
-      - https://data.pyg.org/whl/torch-2.1.0%2Bcu118/torch_sparse-0.6.18%2Bpt21cu118-cp39-cp39-linux_x86_64.whl
-      # Nota: if libcusparse.so.11. errors occur, run
-      # export LD_LIBRARY_PATH="/home/${USER}/miniconda/envs/lib:$LD_LIBRARY_PATH" 
-      # ou
-      # export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/lib:$LD_LIBRARY_PATH" 
-      # see https://github.com/pyg-team/pytorch_geometric/issues/2040#issuecomment-766610625
       # --------- hydra configs --------- #
-      - hydra-core==1.1.*
-      - hydra-colorlog==1.1.*
-      # # --------- Documentation --------- #
-      - sphinx==4.5.*
-      - sphinx_rtd_theme==1.0.*
-      - myst_parser==0.17.*
-      - sphinx_paramlinks==0.5.*
-      - recommonmark==0.7.*
-      - sphinxnotes-mock==1.0.0b0  # still a beta
-      - docutils==0.17
-      - rstcheck==3.3.*  # RST Linter
-      - pyproj==3.6.*
+    - hydra-core==1.1.*
+    - hydra-colorlog==1.1.*
+    # --------- Documentation --------- #
+    - myst_parser==0.17.*
+    - sphinxnotes-mock==1.0.0b0  # still a beta
+    - sphinx_paramlinks==0.5.*