[CI] - Add daily re-caching and log dependency state for pip and conda (

#2107)

* add daily re-caching and log dependency state for pip and conda

* user lower numpy version to fix CI issue

.circleci/config.yml

@@ -17,6 +17,7 @@ commands:
           # OpenEXR requires CMake 3.12. We also download CMake 3.20, which
           # we'll use later for the JS build only.
           command: |
+              echo $(date +%F) > ./date
               echo $(git ls-remote https://github.com/facebookresearch/habitat-lab.git HEAD | awk '{ print $1}') > ./hablab_sha
               cat ./hablab_sha
               wget https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh
@@ -107,7 +108,7 @@ commands:
               if command -v nvidia-smi; then nvidia-smi; fi
       - restore_cache:
           keys:
-            - conda-{{ checksum "habitat-sim/.circleci/config.yml" }}
+            - conda-{{ checksum "habitat-sim/.circleci/config.yml" }}-{{ checksum "./date" }}
       - run: &install_conda
           name: Install conda and dependencies
           no_output_timeout: 20m
@@ -254,11 +255,12 @@ jobs:
       - run:
           name: Combine precommit config and python versions for caching
           command: |
+            echo $(date +%F) > ./date
             cat .pre-commit-config.yaml > pre-commit-deps.txt
             python -VV >> pre-commit-deps.txt
       - restore_cache:
           keys:
-          - v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}
+          - v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}-{{ checksum "./date" }}
 
       - run:
           name: Install Dependencies
@@ -270,7 +272,7 @@ jobs:
       - save_cache:
           paths:
             - ~/.cache/pre-commit
-          key: v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}
+          key: v1-precommit-deps-{{ checksum "pre-commit-deps.txt" }}-{{ checksum "./date" }}
       - run:
           name: Check Code Style using pre-commit
           command: |
@@ -434,6 +436,7 @@ jobs:
       - run:
           name: Install Habitat Sim and Habitat Lab
           command: |
+              echo $(date +%F) > ./date
               export PATH=$HOME/miniconda/bin:$PATH
               . activate habitat;
               cd habitat-sim
@@ -460,7 +463,7 @@ jobs:
               export MULTI_PROC_OFFSET=0 && export MAGNUM_LOG=quiet && export HABITAT_SIM_LOG=quiet
               python -m pytest
       - save_cache:
-          key: habitat-lab-{{ checksum "./hablab_sha" }}
+          key: habitat-lab-{{ checksum "./hablab_sha" }}-{{ checksum "./date" }}
           background: true
           paths:
             - ./habitat-lab
@@ -478,7 +481,7 @@ jobs:
               conda install -y  jinja2 pygments docutils
               ./build-public.sh
       - save_cache:
-          key: conda-{{ checksum "habitat-sim/.circleci/config.yml" }}
+          key: conda-{{ checksum "habitat-sim/.circleci/config.yml" }}-{{ checksum "./date" }}
           background: true
           paths:
             - ~/miniconda
@@ -536,6 +539,23 @@ jobs:
               # switch back to cmake 3.12
               sudo rm /usr/local/bin/cmake
               sudo ln -s /opt/cmake312/bin/cmake /usr/local/bin/cmake
+      - run:
+          name: Display dependency versions
+          no_output_timeout: 20m
+          command: |
+              export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
+              . activate habitat
+              export CONDA_ENV_DUMP=conda_env_dump.log
+              echo "pip freeze"           | tee -a $CONDA_ENV_DUMP
+              pip freeze                  | tee -a $CONDA_ENV_DUMP
+              echo "###########"          | tee -a $CONDA_ENV_DUMP
+              echo "conda list"           | tee -a $CONDA_ENV_DUMP
+              conda list                  | tee -a $CONDA_ENV_DUMP
+              echo "###########"          | tee -a $CONDA_ENV_DUMP
+              echo "habitat-sim commit"   | tee -a $CONDA_ENV_DUMP
+              cat ./hablab_sha            | tee -a $CONDA_ENV_DUMP
+      - store_artifacts:
+          path: conda_env_dump.log  # This is the list of modules in conda and pip
       - run:
           name: Run sim benchmark
           command: |

requirements.txt

@@ -4,7 +4,7 @@ imageio
 imageio-ffmpeg
 matplotlib
 numba
-numpy
+numpy>=1.20.0,<1.24.0
 numpy-quaternion
 pillow
 scipy>=1.3.0