Merge pull request #897 from SyneRBI/bump_cil_modules

update CIL and CCPi-Regularisation version and reinstate tests

CHANGES.md

@@ -1,11 +1,15 @@
 # ChangeLog
 
 ## v3.7.0
-
+- Adds CCPi-Regularisation-Toolkit as prerequisite for CIL and reinstate unit tests
+- Docker:
+  - Handle BUILD_CIL correctly and not install CIL from conda if BUILD_CIL=ON
 - CMake:
   - only pass through `OpenMP*LIBRARY` variables if set, and add OpenMP_iomp_LIBRARY to the list (for Intel compiler) https://github.com/SyneRBI/SIRF-SuperBuild/pull/879
-- updated versions:
+- Updated Versions:
   - SIRF: v3.7.0
+  - CIL: ccf17f393ba911d13b74f2327779dde030098fe6 # 28 May 2024
+  - CCPi-Regularisation-Toolkit: v24.0.1
   - STIR: v6.1.0
 
 ## v3.6.0

Dockerfile

@@ -21,12 +21,16 @@ RUN bash /opt/scripts/build_system-ubuntu.sh
 
 # SIRF python deps
 ARG BUILD_GPU=0
+ARG BUILD_CIL="OFF"
 COPY docker/requirements.yml /opt/scripts/
 # https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.html#conda-environments
 # https://github.com/TomographicImaging/CIL/blob/master/Dockerfile
 RUN if test "$BUILD_GPU" != 0; then \
   sed -ri 's/^(\s*)#\s*(- \S+.*#.*GPU.*)$/\1\2/' /opt/scripts/requirements.yml; \
  fi \
+ && if test "$BUILD_CIL" != "OFF"; then \
+  sed -r -i -e '/^\s*- (cil|ccpi-regulariser).*/d' /opt/scripts/requirements.yml; \
+ fi \
  && conda config --env --set channel_priority strict \
  && for ch in defaults ccpi intel conda-forge; do conda config --env --add channels $ch; done \
  && mamba env update -n base -f /opt/scripts/requirements.yml \
@@ -66,7 +70,8 @@ ARG USE_NiftyPET="OFF"
 ARG BUILD_siemens_to_ismrmrd="ON"
 ARG BUILD_pet_rd_tools="ON"
 ARG Gadgetron_USE_CUDA="ON"
-ARG BUILD_CIL="OFF"
+# BUILD_CIL is defined in the previous stage
+ARG BUILD_CIL
 ARG EXTRA_BUILD_FLAGS=""
 
 # build, install in /opt/SIRF-SuperBuild/{INSTALL,sources/SIRF}, test (if RUN_CTEST)
@@ -99,7 +104,7 @@ RUN apt update -qq && apt install -yq --no-install-recommends \
   && mkdir -p /usr/share/X11/xkb \
   && test -e /usr/bin/X || ln -s /usr/bin/Xorg /usr/bin/X
 
-RUN echo 'test -z "$OMP_NUM_THREADS" && export OMP_NUM_THREADS=$(python -c "import multiprocessing as mc; print(mc.cpu_count() // 2)")' > /usr/local/bin/before-notebook.d/omp_num_threads.sh
+RUN echo 'test -z "$OMP_NUM_THREADS" && export OMP_NUM_THREADS=$(python -c "import multiprocessing as mc; print(max(1, mc.cpu_count() - 2))")' > /usr/local/bin/before-notebook.d/omp_num_threads.sh
 
 COPY --chown=${NB_USER} --chmod=644 --link docker/.bashrc /home/${NB_USER}/
 # RUN sed -i s:PYTHON_INSTALL_DIR:${CONDA_DIR}:g /home/${NB_USER}/.bashrc
@@ -110,8 +115,9 @@ COPY --from=build --link --chown=${NB_USER} /opt/SIRF-SuperBuild/INSTALL/ /opt/S
 #COPY --from=build --link /opt/conda/ /opt/conda/
 
 # install {SIRF-Exercises,CIL-Demos}
+ARG BUILD_CIL
 COPY docker/user_demos.sh /opt/scripts/
-RUN bash /opt/scripts/user_demos.sh \
+RUN BUILD_CIL="${BUILD_CIL}" bash /opt/scripts/user_demos.sh \
  && fix-permissions /opt/SIRF-Exercises /opt/CIL-Demos "${CONDA_DIR}" /home/${NB_USER}
 
 # docker-stacks notebook
@@ -126,4 +132,6 @@ ENV GADGETRON_RELAY_HOST="0.0.0.0"
 
 # run gadgetron in the background before start-notebook.py
 COPY --link --chown=${NB_USER} docker/start-gadgetron-notebook.sh /opt/scripts/
+# COPY --from=build --link --chown=${NB_USER} /opt/SIRF-SuperBuild/INSTALL/lib /opt/conda/lib
+COPY --from=build --link --chown=${NB_USER} /opt/SIRF-SuperBuild/INSTALL/bin/env_sirf.sh /opt/conda/etc/conda/activate.d
 CMD ["/opt/scripts/start-gadgetron-notebook.sh"]

SuperBuild.cmake

@@ -325,7 +325,7 @@ if ("${PYTHON_STRATEGY}" STREQUAL "CONDA")
   set (BUILD_CIL OFF)
 endif()
 if (BUILD_CIL)
-  list(APPEND ${PRIMARY_PROJECT_NAME}_DEPENDENCIES CIL TomoPhantom)
+  list(APPEND ${PRIMARY_PROJECT_NAME}_DEPENDENCIES CIL CCPi-Regularisation-Toolkit TomoPhantom)
 endif()
 if (BUILD_ASTRA)
   list(APPEND ${PRIMARY_PROJECT_NAME}_DEPENDENCIES astra-python-wrapper)

SuperBuild/External_CCPi-Regularisation-Toolkit.cmake

@@ -72,7 +72,8 @@ if(NOT ( DEFINED "USE_SYSTEM_${externalProjName}" AND "${USE_SYSTEM_${externalPr
         -DPYTHON_DEST_DIR:PATH=${PYTHON_DEST}
 
       BUILD_COMMAND ${CMAKE_COMMAND} -E env CIL_VERSION=${${proj}_TAG} ${CMAKE_COMMAND} --build .
-      INSTALL_COMMAND ${CMAKE_COMMAND} -E env CIL_VERSION=${${proj}_TAG} ${CMAKE_COMMAND} --build . --target install
+      INSTALL_COMMAND ${CMAKE_COMMAND} -E env CIL_VERSION=${${proj}_TAG} ${CMAKE_COMMAND} --build . --target install && 
+       ${PYTHON_EXECUTABLE} -m pip install ${${proj}_SOURCE_DIR}/src/Python
       #TEST_COMMAND ${PYTHON_EXECUTABLE} -m unittest discover -s ${${proj}_SOURCE_DIR}/test/ -p test*.py
       DEPENDS
         ${${proj}_DEPENDENCIES}

SuperBuild/External_CIL.cmake

@@ -115,6 +115,9 @@ if(NOT ( DEFINED "USE_SYSTEM_${externalProjName}" AND "${USE_SYSTEM_${externalPr
   add_test(NAME CIL_FRAMEWORK_TESTS_9
            COMMAND ${PYTHON_EXECUTABLE} -m unittest discover -p test_Operator.py
            WORKING_DIRECTORY ${${proj}_SOURCE_DIR}/Wrappers/Python/test)
+  add_test(NAME CIL_FRAMEWORK_TESTS_10
+           COMMAND ${PYTHON_EXECUTABLE} -m unittest discover -p test_PluginsRegularisation.py
+           WORKING_DIRECTORY ${${proj}_SOURCE_DIR}/Wrappers/Python/test)
   add_test(NAME CIL_FRAMEWORK_TESTS_11
            COMMAND ${PYTHON_EXECUTABLE} -m unittest discover -p test_quality_measures.py
            WORKING_DIRECTORY ${${proj}_SOURCE_DIR}/Wrappers/Python/test)
@@ -130,6 +133,7 @@ if(NOT ( DEFINED "USE_SYSTEM_${externalProjName}" AND "${USE_SYSTEM_${externalPr
   add_test(NAME CIL_FRAMEWORK_TESTS_15
            COMMAND ${PYTHON_EXECUTABLE} -m unittest discover -p test_TranslateFunction.py
            WORKING_DIRECTORY ${${proj}_SOURCE_DIR}/Wrappers/Python/test)
+
   # add_test(NAME CIL_FRAMEWORK_TESTS_ALL
   #          COMMAND ${PYTHON_EXECUTABLE} -m unittest discover
   #          WORKING_DIRECTORY ${${proj}_SOURCE_DIR}/Wrappers/Python/test)

docker/requirements.yml

@@ -18,9 +18,17 @@ dependencies:
   - deprecation
   - nose
   - pip
-  - cil              # cil
-  - ccpi-regulariser # cil
-  - dxchange         # cil
+  - cil               # cil
+  - ccpi-regulariser  # cil
+  - dxchange          # cil
+  - h5py              # cil
+  - pillow            # cil
+  - olefile >=0.46    # cil
+  - pywavelets        # cil
+  - cil-data >=21.3.0 # cil
+  - ipp >=2021.10     # cil
+  - tqdm              # cil
+  - numba             # cil 
   # - tigre          # cil (GPU)
   # - astra-toolbox  # cil (GPU)
   - pip:

docker/user_demos.sh

@@ -22,6 +22,15 @@ if [ "$PYTHON" = "miniconda" ]; then
       # delete GPU deps
       sed -r -e '/^\s*- (astra-toolbox|tigre).*/d' -e '/^\s*- \S+.*#.*GPU/d' environment.yml > environment-sirf.yml
     fi
+    # do not install CIL from conda if BUILD_CIL is set
+if test "${BUILD_CIL:-OFF}" != "OFF"; then
+      # delete CIL package from the environment file
+      echo "Deleting CIL from the environment file BUILD_CIL is set to >${BUILD_CIL}<"
+      sed -r -i -e '/^\s*- (cil).*/d' environment-sirf.yml
+      cat environment-sirf.yml
+    else
+      echo "Not deleting CIL from the environment file BUILD_CIL is set to >${BUILD_CIL}<"
+    fi
     conda env update --file environment-sirf.yml
   else
     if [ -f requirements.txt ]; then

version_config.cmake

@@ -161,10 +161,10 @@ set(DEFAULT_JSON_TAG v3.11.3)
 # CCPi CIL
 # minimum supported version of CIL supported is > 22.1.0 or from commit a6062410028c9872c5b355be40b96ed1497fed2a
 set(DEFAULT_CIL_URL https://github.com/TomographicImaging/CIL)
-set(DEFAULT_CIL_TAG db5a2a6cd3bddfbbf53e65f0549ac206096e5b44) # 13 Feb 2024
+set(DEFAULT_CIL_TAG ccf17f393ba911d13b74f2327779dde030098fe6) # 28 May 2024
 
-set(DEFAULT_CCPi-Regularisation-Toolkit_URL https://github.com/vais-ral/CCPi-Regularisation-Toolkit)
-set(DEFAULT_CCPi-Regularisation-Toolkit_TAG "71f8d304d804b54d378f0ed05539f01aaaf13758")
+set(DEFAULT_CCPi-Regularisation-Toolkit_URL https://github.com/TomographicImaging/CCPi-Regularisation-Toolkit)
+set(DEFAULT_CCPi-Regularisation-Toolkit_TAG "v24.0.1")
 
 # CERN ROOT
 set(DEFAULT_ROOT_URL https://github.com/root-project/root)