From 7abaaef889c53a474bcb9ccf37f7d1938525a02f Mon Sep 17 00:00:00 2001 From: Samuel Farrens Date: Thu, 7 Sep 2023 17:42:12 +0200 Subject: [PATCH 01/25] fixed bug in install_shapepipe --- install_shapepipe | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/install_shapepipe b/install_shapepipe index c370d91d0..56fa9be00 100755 --- a/install_shapepipe +++ b/install_shapepipe @@ -200,26 +200,28 @@ check_conda() { echo -ne "${RED}ERROR: Found Conda version $CONDA_VERSION but require 4.0.0 or greater.${NC}\n" exit 1 fi - # Check Conda minor version - if [ "$CONDA_VERSION_MINOR" -gt 4 ] + # Look for conda.sh file + if [ -f "$CONDA_PREFIX_1$CONDA_SH" ] then - if [ -f "$CONDA_PREFIX_1$CONDA_SH" ] - then - source "$CONDA_PREFIX_1$CONDA_SH" - elif [ -f "$CONDA_PREFIX$CONDA_SH" ] - then - source "$CONDA_PREFIX$CONDA_SH" - else - echo -ne "${RED}ERROR: Could not find $CONDA_SH in \$CONDA_PREFIX.${NC}\n" - echo -ne "${RED}Activate the base/root Conda environment and try again.${NC}\n" - exit 1 - fi - CONDA_ACTIVATE="conda activate" - CONDA_DEACTIVATE="conda deactivate" + source "$CONDA_PREFIX_1$CONDA_SH" + elif [ -f "$CONDA_PREFIX$CONDA_SH" ] + then + source "$CONDA_PREFIX$CONDA_SH" else + echo -ne "${RED}ERROR: Could not find $CONDA_SH in \$CONDA_PREFIX.${NC}\n" + echo -ne "${RED}Activate the base/root Conda environment and try again.${NC}\n" + exit 1 + fi + # Set environment activation command + if [ "$USE_CONDA_SOURCE" == TRUE ] + then CONDA_ACTIVATE="source activate" CONDA_DEACTIVATE="source deactivate" + else + CONDA_ACTIVATE="conda activate" + CONDA_DEACTIVATE="conda deactivate" fi + } # Check if a binary executable is already installed in the conda environment @@ -464,6 +466,7 @@ Options:\n \t--pipe-dir\t set the path to the ShapePipe base directory (default is \$PWD)\n \t--build-dir\t set the path to the ShapePipe build (default is \$PWD/build)\n \t--env-name\t set the name of the conda environment (default is shapepipe)\n +\t--use-conda-source\t use conda source instead of the default conda activate to activate the shapepipe environment\n \t--develop\t option to install additional development packages\n \t--vos\t\t option to install virtual observatory software\n \t--no-env\t do not build Conda environment\n @@ -525,6 +528,10 @@ case $i in ENV_NAME="${i#*=}" shift ;; + --use-conda-source) + USE_CONDA_SOURCE=TRUE + shift + ;; --develop) DEVELOP=TRUE shift From 8a4ab6437c8de473615f122b37e620d563d5acff Mon Sep 17 00:00:00 2001 From: sfarrens Date: Wed, 8 Nov 2023 17:34:20 +0100 Subject: [PATCH 02/25] Added development environment. --- .github/workflows/ci-dev.yml | 50 +++++++++++++++++++ .../{ci-tests.yml => ci-release.yml} | 1 - environment-dev.yml | 37 ++++++++++++++ environment.yml | 6 +++ install_shapepipe | 34 +++++++++++-- 5 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/ci-dev.yml rename .github/workflows/{ci-tests.yml => ci-release.yml} (98%) create mode 100644 environment-dev.yml diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml new file mode 100644 index 000000000..a260f6c9d --- /dev/null +++ b/.github/workflows/ci-dev.yml @@ -0,0 +1,50 @@ +name: CI + +on: + pull_request: + branches: + - develop + +jobs: + + test-full: + name: Full Test Suite + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + python-version: [3.8] + + steps: + - uses: actions/checkout@v2 + + - name: Install Linux dependencies + if: runner.os == 'Linux' + run: sudo apt-get install libopenblas-dev + + - name: Install macOS Dependencies + shell: bash -l {0} + if: runner.os == 'macOS' + run: | + brew install libomp + + - name: Set up conda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + auto-activate-base: true + + - name: Install package + shell: bash -l {0} + run: | + ./install_shapepipe --env-dev --develop + + - name: Run tests + shell: bash -l {0} + run: | + conda activate shapepipe + python setup.py test + shapepipe_run -c example/config.ini diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-release.yml similarity index 98% rename from .github/workflows/ci-tests.yml rename to .github/workflows/ci-release.yml index cc4ae4e93..ef1f6a013 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-release.yml @@ -5,7 +5,6 @@ on: branches: - main - master - - develop jobs: diff --git a/environment-dev.yml b/environment-dev.yml new file mode 100644 index 000000000..b5325d02c --- /dev/null +++ b/environment-dev.yml @@ -0,0 +1,37 @@ +# ShapePipe Development Environment +# --------------------------------- +# This environment should only be used for development and testing of +# ShapePipe (i.e. nothing that could be published). Packages need only +# specify the minimum compatible versions. This environment can updated +# as needed. +name: shapepipe-dev +channels: + - conda-forge +dependencies: + - python=3.11 + - pip>=21.2.4 + - astropy>=5.0 + - autoconf>=2.71 + - automake>=1.16 + - cmake>=3.27 + - galsim>=2.5 + - joblib>=1.3 + - libtool>=2.4 + - matplotlib>=3.8 + - numba>=0.57 + - pandas>=2.1 + - pyqt5-sip>=12.12 + - pyqtgraph>=0.13 + - reproject>=0.12 + - sqlitedict>=2.1 + - termcolor>=2.3 + - tqdm>=4.66 + - treecorr>=4.3 + - pip: + - mccd==1.2.3 + - modopt==1.6.0 + - sip_tpv==1.1 + - sf_tools==2.0.4 + - git+https://github.com/CEA-COSMIC/pysap@develop + - git+https://github.com/aguinot/ngmix@stable_version + - git+https://github.com/tobias-liaudat/Stile@v0.1 diff --git a/environment.yml b/environment.yml index 9279402ef..ec8fae7a1 100644 --- a/environment.yml +++ b/environment.yml @@ -1,3 +1,9 @@ +# ShapePipe Production Environment +# -------------------------------- +# This environment should be used for any production runs of ShapePipe +# (i.e. anything that could be published). All packages are pinned to +# specific versions to maximise the reproducibility of the results. +# This environment should be updated for each release of ShapePipe. name: shapepipe channels: - conda-forge diff --git a/install_shapepipe b/install_shapepipe index 56fa9be00..4c3f512c6 100755 --- a/install_shapepipe +++ b/install_shapepipe @@ -81,6 +81,8 @@ NC='\033[0m' PIPE_DIR=$PWD BUILD_DIR=$PIPE_DIR/build ENV_NAME='shapepipe' +ENV_DEV=FALSE +USE_CONDA_SOURCE=FALSE BUILD_ENV=TRUE DEVELOP=FALSE VOS=FALSE @@ -258,6 +260,14 @@ check_binary2() { echo "" } +# Check the environment name +check_env_name() { +if [ "$ENV_DEV" == TRUE ] +then + ENV_NAME='shapepipe-dev' +fi +} + # Check if a python site-package is installed correctly in the conda environment check_site_pkg() { if compgen -G "$CONDA_PREFIX/lib/python3.*/site-packages/$1*" > /dev/null @@ -343,7 +353,9 @@ package_report() { echo 'Package Report' echo '' packages - python -m piplist + echo 'Python Package Check List:' + echo '' + site-packages echo '' exit_code } @@ -410,6 +422,7 @@ astromatic_setup() { # Function to uninstall ShapePipe and dependencies uninstall() { + check_env_name read -p "Uninstall $ENV_NAME environment [y/N]: " response response=${response:-n} if [ $response == "y" ] @@ -465,8 +478,9 @@ Options:\n \t--uninstall\t uninstall ShapePipe\n \t--pipe-dir\t set the path to the ShapePipe base directory (default is \$PWD)\n \t--build-dir\t set the path to the ShapePipe build (default is \$PWD/build)\n -\t--env-name\t set the name of the conda environment (default is shapepipe)\n -\t--use-conda-source\t use conda source instead of the default conda activate to activate the shapepipe environment\n +\t--env-name\t set the name of the Conda environment (default is shapepipe)\n +\t--env-dev\t use the developers Conda environment instead of the standard production environment\n +\t--use-conda-source\t use 'conda source' instead of the default 'conda activate' to activate the shapepipe environment\n \t--develop\t option to install additional development packages\n \t--vos\t\t option to install virtual observatory software\n \t--no-env\t do not build Conda environment\n @@ -528,6 +542,10 @@ case $i in ENV_NAME="${i#*=}" shift ;; + --env-dev) + ENV_DEV=TRUE + shift + ;; --use-conda-source) USE_CONDA_SOURCE=TRUE shift @@ -620,6 +638,9 @@ start # Check if conda is installed check_conda +# Check the Conda environment name +check_env_name + # Find the operating system case "$OSTYPE" in darwin*) @@ -652,7 +673,12 @@ setup if [ "$BUILD_ENV" == TRUE ] then report_progress 'ShapePipe Environment' - conda env create -f environment.yml -n $ENV_NAME + if [ "$ENV_DEV" == TRUE ] + then + conda env create -f environment-dev.yml -n $ENV_NAME + else + conda env create -f environment.yml -n $ENV_NAME + fi fi # Activate conda environment From 943388ffafeed1ab75f8e47d23803a4fcfc54929 Mon Sep 17 00:00:00 2001 From: Samuel Farrens Date: Thu, 30 Nov 2023 17:48:04 +0100 Subject: [PATCH 03/25] added BigMac depdendency for macOS --- .github/workflows/ci-dev.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index a260f6c9d..984a9e7d6 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -28,7 +28,8 @@ jobs: shell: bash -l {0} if: runner.os == 'macOS' run: | - brew install libomp + brew tap sfarrens/sf + brew install bigmac libomp - name: Set up conda uses: conda-incubator/setup-miniconda@v2 From d3740b175f4f3f96f1d9b511e345e3d48ec8b1ac Mon Sep 17 00:00:00 2001 From: sfarrens Date: Fri, 1 Dec 2023 10:05:55 +0100 Subject: [PATCH 04/25] activate dev environment --- .github/workflows/ci-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index 984a9e7d6..9e666f776 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -46,6 +46,6 @@ jobs: - name: Run tests shell: bash -l {0} run: | - conda activate shapepipe + conda activate shapepipe-dev python setup.py test shapepipe_run -c example/config.ini From 16f58d96bcddebc5f5091dced6a08796379dc0e8 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Fri, 1 Dec 2023 13:11:59 +0100 Subject: [PATCH 05/25] added cs_util to dev env --- environment-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment-dev.yml b/environment-dev.yml index b5325d02c..cd2977ab8 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -28,6 +28,7 @@ dependencies: - tqdm>=4.66 - treecorr>=4.3 - pip: + - cs_util==0.0.5 - mccd==1.2.3 - modopt==1.6.0 - sip_tpv==1.1 From e31cd7cc07b55b59e7ee14ab0ec35f66a76a1889 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Thu, 10 Oct 2024 18:08:49 +0200 Subject: [PATCH 06/25] unpinned pip dependencies --- environment-dev.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index cd2977ab8..95f8db62b 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -28,11 +28,11 @@ dependencies: - tqdm>=4.66 - treecorr>=4.3 - pip: - - cs_util==0.0.5 - - mccd==1.2.3 - - modopt==1.6.0 - - sip_tpv==1.1 - - sf_tools==2.0.4 + - cs_util>=0.0.5 + - mccd>=1.2.3 + - modopt>=1.6 + - sip_tpv>=1.1 + - sf_tools>=2.0.4 - git+https://github.com/CEA-COSMIC/pysap@develop - git+https://github.com/aguinot/ngmix@stable_version - git+https://github.com/tobias-liaudat/Stile@v0.1 From 1462605361888af655667dd75a8f50c41823fcb0 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Fri, 11 Oct 2024 17:10:50 +0200 Subject: [PATCH 07/25] updated install setup --- .github/workflows/ci-dev.yml | 7 +++-- pyproject.toml | 29 +++++++++++++++++++ setup.cfg => setup.cfg.old | 0 setup.py => setup.py.old | 0 {shapepipe => src/shapepipe}/__init__.py | 5 +++- {shapepipe => src/shapepipe}/info.py | 29 ++----------------- .../shapepipe}/modules/__init__.py | 0 .../modules/execute_example_runner.py | 0 .../find_exposures_package/__init__.py | 0 .../find_exposures_package/find_exposures.py | 0 .../modules/find_exposures_runner.py | 0 .../modules/get_images_package/__init__.py | 0 .../modules/get_images_package/get_images.py | 0 .../shapepipe}/modules/get_images_runner.py | 0 .../modules/make_cat_package/__init__.py | 0 .../modules/make_cat_package/make_cat.py | 0 .../shapepipe}/modules/make_cat_runner.py | 0 .../modules/mask_package/__init__.py | 0 .../shapepipe}/modules/mask_package/mask.py | 0 .../shapepipe}/modules/mask_runner.py | 0 .../match_external_package/__init__.py | 0 .../match_external_package/match_external.py | 0 .../modules/match_external_runner.py | 0 .../shapepipe}/modules/mccd_fit_runner.py | 0 .../shapepipe}/modules/mccd_fit_val_runner.py | 0 .../shapepipe}/modules/mccd_interp_runner.py | 0 .../modules/mccd_package/__init__.py | 0 .../mccd_package/mccd_interpolation_script.py | 0 .../mccd_package/mccd_plot_utilities.py | 0 .../mccd_package/shapepipe_auxiliary_mccd.py | 0 .../shapepipe}/modules/mccd_plots_runner.py | 0 .../modules/mccd_preprocessing_runner.py | 0 .../shapepipe}/modules/mccd_val_runner.py | 0 .../modules/merge_headers_package/__init__.py | 0 .../merge_headers_package/merge_headers.py | 0 .../modules/merge_headers_runner.py | 0 .../merge_sep_cats_package/__init__.py | 0 .../merge_sep_cats_package/merge_sep_cats.py | 0 .../modules/merge_sep_cats_runner.py | 0 .../modules/merge_starcat_package/__init__.py | 0 .../merge_starcat_package/merge_starcat.py | 0 .../modules/merge_starcat_runner.py | 0 .../shapepipe}/modules/module_decorator.py | 0 .../shapepipe}/modules/module_runners.py | 0 .../modules/ngmix_package/__init__.py | 0 .../shapepipe}/modules/ngmix_package/ngmix.py | 0 .../shapepipe}/modules/ngmix_runner.py | 0 .../modules/pastecat_package/__init__.py | 0 .../modules/pastecat_package/pastecat.py | 0 .../shapepipe}/modules/pastecat_runner.py | 0 .../modules/psfex_interp_package/__init__.py | 0 .../psfex_interp_package/psfex_interp.py | 0 .../shapepipe}/modules/psfex_interp_runner.py | 0 .../modules/psfex_package/__init__.py | 0 .../modules/psfex_package/psfex_script.py | 0 .../shapepipe}/modules/psfex_runner.py | 0 .../python_example_package/__init__.py | 0 .../python_example_package/python_example.py | 0 .../modules/python_example_runner.py | 0 .../modules/random_cat_package/__init__.py | 0 .../modules/random_cat_package/random_cat.py | 0 .../shapepipe}/modules/random_cat_runner.py | 0 .../modules/serial_example_runner.py | 0 .../modules/setools_package/__init__.py | 0 .../modules/setools_package/setools.py | 0 .../shapepipe}/modules/setools_runner.py | 0 .../modules/sextractor_package/__init__.py | 0 .../sextractor_package/sextractor_script.py | 0 .../shapepipe}/modules/sextractor_runner.py | 0 .../modules/split_exp_package/__init__.py | 0 .../modules/split_exp_package/split_exp.py | 0 .../shapepipe}/modules/split_exp_runner.py | 0 .../modules/spread_model_package/__init__.py | 0 .../spread_model_package/spread_model.py | 0 .../shapepipe}/modules/spread_model_runner.py | 0 .../uncompress_fits_package/__init__.py | 0 .../uncompress_fits.py | 0 .../modules/uncompress_fits_runner.py | 0 .../modules/vignetmaker_package/__init__.py | 0 .../vignetmaker_package/vignetmaker.py | 0 .../shapepipe}/modules/vignetmaker_runner.py | 0 .../shapepipe}/pipeline/__init__.py | 0 {shapepipe => src/shapepipe}/pipeline/args.py | 3 +- .../shapepipe}/pipeline/config.py | 0 .../shapepipe}/pipeline/dependency_handler.py | 0 .../shapepipe}/pipeline/execute.py | 0 .../shapepipe}/pipeline/file_handler.py | 0 .../shapepipe}/pipeline/file_io.py | 0 .../shapepipe}/pipeline/job_handler.py | 0 .../shapepipe}/pipeline/mpi_run.py | 0 .../shapepipe}/pipeline/run_log.py | 0 .../shapepipe}/pipeline/shared.py | 0 .../shapepipe}/pipeline/str_handler.py | 0 .../shapepipe}/pipeline/timeout.py | 0 .../shapepipe}/pipeline/worker_handler.py | 0 {shapepipe => src/shapepipe}/run.py | 7 +++-- .../python => src/shapepipe}/shapepipe_run.py | 0 .../shapepipe}/tests/test_get_images.py | 0 .../shapepipe}/tests/test_pipeline.py | 0 .../shapepipe}/tests/test_utilities.py | 0 .../shapepipe}/utilities/__init__.py | 0 .../shapepipe}/utilities/cfis.py | 0 .../shapepipe}/utilities/file_system.py | 0 .../shapepipe}/utilities/galaxy.py | 0 104 files changed, 45 insertions(+), 35 deletions(-) create mode 100644 pyproject.toml rename setup.cfg => setup.cfg.old (100%) rename setup.py => setup.py.old (100%) rename {shapepipe => src/shapepipe}/__init__.py (59%) rename {shapepipe => src/shapepipe}/info.py (76%) rename {shapepipe => src/shapepipe}/modules/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/execute_example_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/find_exposures_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/find_exposures_package/find_exposures.py (100%) rename {shapepipe => src/shapepipe}/modules/find_exposures_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/get_images_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/get_images_package/get_images.py (100%) rename {shapepipe => src/shapepipe}/modules/get_images_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/make_cat_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/make_cat_package/make_cat.py (100%) rename {shapepipe => src/shapepipe}/modules/make_cat_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/mask_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/mask_package/mask.py (100%) rename {shapepipe => src/shapepipe}/modules/mask_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/match_external_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/match_external_package/match_external.py (100%) rename {shapepipe => src/shapepipe}/modules/match_external_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/mccd_fit_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/mccd_fit_val_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/mccd_interp_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/mccd_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/mccd_package/mccd_interpolation_script.py (100%) rename {shapepipe => src/shapepipe}/modules/mccd_package/mccd_plot_utilities.py (100%) rename {shapepipe => src/shapepipe}/modules/mccd_package/shapepipe_auxiliary_mccd.py (100%) rename {shapepipe => src/shapepipe}/modules/mccd_plots_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/mccd_preprocessing_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/mccd_val_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/merge_headers_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/merge_headers_package/merge_headers.py (100%) rename {shapepipe => src/shapepipe}/modules/merge_headers_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/merge_sep_cats_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/merge_sep_cats_package/merge_sep_cats.py (100%) rename {shapepipe => src/shapepipe}/modules/merge_sep_cats_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/merge_starcat_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/merge_starcat_package/merge_starcat.py (100%) rename {shapepipe => src/shapepipe}/modules/merge_starcat_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/module_decorator.py (100%) rename {shapepipe => src/shapepipe}/modules/module_runners.py (100%) rename {shapepipe => src/shapepipe}/modules/ngmix_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/ngmix_package/ngmix.py (100%) rename {shapepipe => src/shapepipe}/modules/ngmix_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/pastecat_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/pastecat_package/pastecat.py (100%) rename {shapepipe => src/shapepipe}/modules/pastecat_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/psfex_interp_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/psfex_interp_package/psfex_interp.py (100%) rename {shapepipe => src/shapepipe}/modules/psfex_interp_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/psfex_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/psfex_package/psfex_script.py (100%) rename {shapepipe => src/shapepipe}/modules/psfex_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/python_example_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/python_example_package/python_example.py (100%) rename {shapepipe => src/shapepipe}/modules/python_example_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/random_cat_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/random_cat_package/random_cat.py (100%) rename {shapepipe => src/shapepipe}/modules/random_cat_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/serial_example_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/setools_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/setools_package/setools.py (100%) rename {shapepipe => src/shapepipe}/modules/setools_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/sextractor_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/sextractor_package/sextractor_script.py (100%) rename {shapepipe => src/shapepipe}/modules/sextractor_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/split_exp_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/split_exp_package/split_exp.py (100%) rename {shapepipe => src/shapepipe}/modules/split_exp_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/spread_model_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/spread_model_package/spread_model.py (100%) rename {shapepipe => src/shapepipe}/modules/spread_model_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/uncompress_fits_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/uncompress_fits_package/uncompress_fits.py (100%) rename {shapepipe => src/shapepipe}/modules/uncompress_fits_runner.py (100%) rename {shapepipe => src/shapepipe}/modules/vignetmaker_package/__init__.py (100%) rename {shapepipe => src/shapepipe}/modules/vignetmaker_package/vignetmaker.py (100%) rename {shapepipe => src/shapepipe}/modules/vignetmaker_runner.py (100%) rename {shapepipe => src/shapepipe}/pipeline/__init__.py (100%) rename {shapepipe => src/shapepipe}/pipeline/args.py (97%) rename {shapepipe => src/shapepipe}/pipeline/config.py (100%) rename {shapepipe => src/shapepipe}/pipeline/dependency_handler.py (100%) rename {shapepipe => src/shapepipe}/pipeline/execute.py (100%) rename {shapepipe => src/shapepipe}/pipeline/file_handler.py (100%) rename {shapepipe => src/shapepipe}/pipeline/file_io.py (100%) rename {shapepipe => src/shapepipe}/pipeline/job_handler.py (100%) rename {shapepipe => src/shapepipe}/pipeline/mpi_run.py (100%) rename {shapepipe => src/shapepipe}/pipeline/run_log.py (100%) rename {shapepipe => src/shapepipe}/pipeline/shared.py (100%) rename {shapepipe => src/shapepipe}/pipeline/str_handler.py (100%) rename {shapepipe => src/shapepipe}/pipeline/timeout.py (100%) rename {shapepipe => src/shapepipe}/pipeline/worker_handler.py (100%) rename {shapepipe => src/shapepipe}/run.py (98%) rename {scripts/python => src/shapepipe}/shapepipe_run.py (100%) rename {shapepipe => src/shapepipe}/tests/test_get_images.py (100%) rename {shapepipe => src/shapepipe}/tests/test_pipeline.py (100%) rename {shapepipe => src/shapepipe}/tests/test_utilities.py (100%) rename {shapepipe => src/shapepipe}/utilities/__init__.py (100%) rename {shapepipe => src/shapepipe}/utilities/cfis.py (100%) rename {shapepipe => src/shapepipe}/utilities/file_system.py (100%) rename {shapepipe => src/shapepipe}/utilities/galaxy.py (100%) diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index 9e666f776..1ce5c0070 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -15,10 +15,10 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python-version: [3.8] + python-version: [3.11] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout - name: Install Linux dependencies if: runner.os == 'Linux' @@ -37,6 +37,7 @@ jobs: auto-update-conda: true python-version: ${{ matrix.python-version }} auto-activate-base: true + miniforge-version: latest - name: Install package shell: bash -l {0} @@ -47,5 +48,5 @@ jobs: shell: bash -l {0} run: | conda activate shapepipe-dev - python setup.py test + pytest shapepipe_run -c example/config.ini diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..b038eb1cd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "shapepipe" +version = "1.0.2" +description = "Galaxy shape measurement pipeline." +authors = [ + { name = "Samuel Farrens", email = "samuel.farrens@cea.fr" }, + { name = "Axel Guinot", email = "axel.guinot@cea.fr" }, + { name = "Martin Kilbinger", email = "martin.kilbinger@cea.fr" } +] +dependencies = [ + "joblib>=0.13", + "modopt>=1.2", + "numpy>=1.14" +] + +[project.optional-dependencies] +testing = [ + "pytest", + "pytest-cov", + "pytest-pycodestyle", + "pytest-pydocstyle" +] + +[project.scripts] +shapepipe_run = "shapepipe.shapepipe_run:main" + +[tool.pytest.ini_options] +addopts = "--verbose --cov=shapepipe" +testpaths = ["shapepipe"] \ No newline at end of file diff --git a/setup.cfg b/setup.cfg.old similarity index 100% rename from setup.cfg rename to setup.cfg.old diff --git a/setup.py b/setup.py.old similarity index 100% rename from setup.py rename to setup.py.old diff --git a/shapepipe/__init__.py b/src/shapepipe/__init__.py similarity index 59% rename from shapepipe/__init__.py rename to src/shapepipe/__init__.py index 88d7a1dfd..a7b698f11 100644 --- a/shapepipe/__init__.py +++ b/src/shapepipe/__init__.py @@ -8,5 +8,8 @@ __all__ = ['modules', 'pipeline', 'utilities'] +from importlib.metadata import metadata, version from . import * -from .info import __about__, __version__ + +__version__ = version("shapepipe") +__about__ = metadata("shapepipe").get("Summary") \ No newline at end of file diff --git a/shapepipe/info.py b/src/shapepipe/info.py similarity index 76% rename from shapepipe/info.py rename to src/shapepipe/info.py index d230bcd7f..b22d9e09f 100644 --- a/shapepipe/info.py +++ b/src/shapepipe/info.py @@ -12,33 +12,8 @@ import_fail = True else: import_fail = False - - -# Package Info -version_info = (1, 0, 1) -__version__ = '.'.join(str(c) for c in version_info) -__name__ = 'shapepipe' -__author__ = 'Samuel Farrens' -__email__ = 'samuel.farrens@cea.fr' -__about__ = ( - 'ShapePipe is a shape measurement pipeline developed with the' - + 'CosmoStat lab at CEA Paris-Saclay.' -) -__setups__ = ['pytest-runner'] -__installs__ = [ - 'joblib>=0.13', - 'modopt>=1.2', - 'numpy>=1.14' -] -__tests__ = [ - 'pytest', - 'pytest-cov', - 'pytest-pycodestyle', - 'pytest-pydocstyle', -] -__scripts_dir__ = 'scripts' -__scripts_ext__ = ('.py', '.sh', '.bash') - + +from shapepipe import __version__ def shapepipe_logo(colour=False): """Get ShapePipe Logo. diff --git a/shapepipe/modules/__init__.py b/src/shapepipe/modules/__init__.py similarity index 100% rename from shapepipe/modules/__init__.py rename to src/shapepipe/modules/__init__.py diff --git a/shapepipe/modules/execute_example_runner.py b/src/shapepipe/modules/execute_example_runner.py similarity index 100% rename from shapepipe/modules/execute_example_runner.py rename to src/shapepipe/modules/execute_example_runner.py diff --git a/shapepipe/modules/find_exposures_package/__init__.py b/src/shapepipe/modules/find_exposures_package/__init__.py similarity index 100% rename from shapepipe/modules/find_exposures_package/__init__.py rename to src/shapepipe/modules/find_exposures_package/__init__.py diff --git a/shapepipe/modules/find_exposures_package/find_exposures.py b/src/shapepipe/modules/find_exposures_package/find_exposures.py similarity index 100% rename from shapepipe/modules/find_exposures_package/find_exposures.py rename to src/shapepipe/modules/find_exposures_package/find_exposures.py diff --git a/shapepipe/modules/find_exposures_runner.py b/src/shapepipe/modules/find_exposures_runner.py similarity index 100% rename from shapepipe/modules/find_exposures_runner.py rename to src/shapepipe/modules/find_exposures_runner.py diff --git a/shapepipe/modules/get_images_package/__init__.py b/src/shapepipe/modules/get_images_package/__init__.py similarity index 100% rename from shapepipe/modules/get_images_package/__init__.py rename to src/shapepipe/modules/get_images_package/__init__.py diff --git a/shapepipe/modules/get_images_package/get_images.py b/src/shapepipe/modules/get_images_package/get_images.py similarity index 100% rename from shapepipe/modules/get_images_package/get_images.py rename to src/shapepipe/modules/get_images_package/get_images.py diff --git a/shapepipe/modules/get_images_runner.py b/src/shapepipe/modules/get_images_runner.py similarity index 100% rename from shapepipe/modules/get_images_runner.py rename to src/shapepipe/modules/get_images_runner.py diff --git a/shapepipe/modules/make_cat_package/__init__.py b/src/shapepipe/modules/make_cat_package/__init__.py similarity index 100% rename from shapepipe/modules/make_cat_package/__init__.py rename to src/shapepipe/modules/make_cat_package/__init__.py diff --git a/shapepipe/modules/make_cat_package/make_cat.py b/src/shapepipe/modules/make_cat_package/make_cat.py similarity index 100% rename from shapepipe/modules/make_cat_package/make_cat.py rename to src/shapepipe/modules/make_cat_package/make_cat.py diff --git a/shapepipe/modules/make_cat_runner.py b/src/shapepipe/modules/make_cat_runner.py similarity index 100% rename from shapepipe/modules/make_cat_runner.py rename to src/shapepipe/modules/make_cat_runner.py diff --git a/shapepipe/modules/mask_package/__init__.py b/src/shapepipe/modules/mask_package/__init__.py similarity index 100% rename from shapepipe/modules/mask_package/__init__.py rename to src/shapepipe/modules/mask_package/__init__.py diff --git a/shapepipe/modules/mask_package/mask.py b/src/shapepipe/modules/mask_package/mask.py similarity index 100% rename from shapepipe/modules/mask_package/mask.py rename to src/shapepipe/modules/mask_package/mask.py diff --git a/shapepipe/modules/mask_runner.py b/src/shapepipe/modules/mask_runner.py similarity index 100% rename from shapepipe/modules/mask_runner.py rename to src/shapepipe/modules/mask_runner.py diff --git a/shapepipe/modules/match_external_package/__init__.py b/src/shapepipe/modules/match_external_package/__init__.py similarity index 100% rename from shapepipe/modules/match_external_package/__init__.py rename to src/shapepipe/modules/match_external_package/__init__.py diff --git a/shapepipe/modules/match_external_package/match_external.py b/src/shapepipe/modules/match_external_package/match_external.py similarity index 100% rename from shapepipe/modules/match_external_package/match_external.py rename to src/shapepipe/modules/match_external_package/match_external.py diff --git a/shapepipe/modules/match_external_runner.py b/src/shapepipe/modules/match_external_runner.py similarity index 100% rename from shapepipe/modules/match_external_runner.py rename to src/shapepipe/modules/match_external_runner.py diff --git a/shapepipe/modules/mccd_fit_runner.py b/src/shapepipe/modules/mccd_fit_runner.py similarity index 100% rename from shapepipe/modules/mccd_fit_runner.py rename to src/shapepipe/modules/mccd_fit_runner.py diff --git a/shapepipe/modules/mccd_fit_val_runner.py b/src/shapepipe/modules/mccd_fit_val_runner.py similarity index 100% rename from shapepipe/modules/mccd_fit_val_runner.py rename to src/shapepipe/modules/mccd_fit_val_runner.py diff --git a/shapepipe/modules/mccd_interp_runner.py b/src/shapepipe/modules/mccd_interp_runner.py similarity index 100% rename from shapepipe/modules/mccd_interp_runner.py rename to src/shapepipe/modules/mccd_interp_runner.py diff --git a/shapepipe/modules/mccd_package/__init__.py b/src/shapepipe/modules/mccd_package/__init__.py similarity index 100% rename from shapepipe/modules/mccd_package/__init__.py rename to src/shapepipe/modules/mccd_package/__init__.py diff --git a/shapepipe/modules/mccd_package/mccd_interpolation_script.py b/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py similarity index 100% rename from shapepipe/modules/mccd_package/mccd_interpolation_script.py rename to src/shapepipe/modules/mccd_package/mccd_interpolation_script.py diff --git a/shapepipe/modules/mccd_package/mccd_plot_utilities.py b/src/shapepipe/modules/mccd_package/mccd_plot_utilities.py similarity index 100% rename from shapepipe/modules/mccd_package/mccd_plot_utilities.py rename to src/shapepipe/modules/mccd_package/mccd_plot_utilities.py diff --git a/shapepipe/modules/mccd_package/shapepipe_auxiliary_mccd.py b/src/shapepipe/modules/mccd_package/shapepipe_auxiliary_mccd.py similarity index 100% rename from shapepipe/modules/mccd_package/shapepipe_auxiliary_mccd.py rename to src/shapepipe/modules/mccd_package/shapepipe_auxiliary_mccd.py diff --git a/shapepipe/modules/mccd_plots_runner.py b/src/shapepipe/modules/mccd_plots_runner.py similarity index 100% rename from shapepipe/modules/mccd_plots_runner.py rename to src/shapepipe/modules/mccd_plots_runner.py diff --git a/shapepipe/modules/mccd_preprocessing_runner.py b/src/shapepipe/modules/mccd_preprocessing_runner.py similarity index 100% rename from shapepipe/modules/mccd_preprocessing_runner.py rename to src/shapepipe/modules/mccd_preprocessing_runner.py diff --git a/shapepipe/modules/mccd_val_runner.py b/src/shapepipe/modules/mccd_val_runner.py similarity index 100% rename from shapepipe/modules/mccd_val_runner.py rename to src/shapepipe/modules/mccd_val_runner.py diff --git a/shapepipe/modules/merge_headers_package/__init__.py b/src/shapepipe/modules/merge_headers_package/__init__.py similarity index 100% rename from shapepipe/modules/merge_headers_package/__init__.py rename to src/shapepipe/modules/merge_headers_package/__init__.py diff --git a/shapepipe/modules/merge_headers_package/merge_headers.py b/src/shapepipe/modules/merge_headers_package/merge_headers.py similarity index 100% rename from shapepipe/modules/merge_headers_package/merge_headers.py rename to src/shapepipe/modules/merge_headers_package/merge_headers.py diff --git a/shapepipe/modules/merge_headers_runner.py b/src/shapepipe/modules/merge_headers_runner.py similarity index 100% rename from shapepipe/modules/merge_headers_runner.py rename to src/shapepipe/modules/merge_headers_runner.py diff --git a/shapepipe/modules/merge_sep_cats_package/__init__.py b/src/shapepipe/modules/merge_sep_cats_package/__init__.py similarity index 100% rename from shapepipe/modules/merge_sep_cats_package/__init__.py rename to src/shapepipe/modules/merge_sep_cats_package/__init__.py diff --git a/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py b/src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py similarity index 100% rename from shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py rename to src/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py diff --git a/shapepipe/modules/merge_sep_cats_runner.py b/src/shapepipe/modules/merge_sep_cats_runner.py similarity index 100% rename from shapepipe/modules/merge_sep_cats_runner.py rename to src/shapepipe/modules/merge_sep_cats_runner.py diff --git a/shapepipe/modules/merge_starcat_package/__init__.py b/src/shapepipe/modules/merge_starcat_package/__init__.py similarity index 100% rename from shapepipe/modules/merge_starcat_package/__init__.py rename to src/shapepipe/modules/merge_starcat_package/__init__.py diff --git a/shapepipe/modules/merge_starcat_package/merge_starcat.py b/src/shapepipe/modules/merge_starcat_package/merge_starcat.py similarity index 100% rename from shapepipe/modules/merge_starcat_package/merge_starcat.py rename to src/shapepipe/modules/merge_starcat_package/merge_starcat.py diff --git a/shapepipe/modules/merge_starcat_runner.py b/src/shapepipe/modules/merge_starcat_runner.py similarity index 100% rename from shapepipe/modules/merge_starcat_runner.py rename to src/shapepipe/modules/merge_starcat_runner.py diff --git a/shapepipe/modules/module_decorator.py b/src/shapepipe/modules/module_decorator.py similarity index 100% rename from shapepipe/modules/module_decorator.py rename to src/shapepipe/modules/module_decorator.py diff --git a/shapepipe/modules/module_runners.py b/src/shapepipe/modules/module_runners.py similarity index 100% rename from shapepipe/modules/module_runners.py rename to src/shapepipe/modules/module_runners.py diff --git a/shapepipe/modules/ngmix_package/__init__.py b/src/shapepipe/modules/ngmix_package/__init__.py similarity index 100% rename from shapepipe/modules/ngmix_package/__init__.py rename to src/shapepipe/modules/ngmix_package/__init__.py diff --git a/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py similarity index 100% rename from shapepipe/modules/ngmix_package/ngmix.py rename to src/shapepipe/modules/ngmix_package/ngmix.py diff --git a/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py similarity index 100% rename from shapepipe/modules/ngmix_runner.py rename to src/shapepipe/modules/ngmix_runner.py diff --git a/shapepipe/modules/pastecat_package/__init__.py b/src/shapepipe/modules/pastecat_package/__init__.py similarity index 100% rename from shapepipe/modules/pastecat_package/__init__.py rename to src/shapepipe/modules/pastecat_package/__init__.py diff --git a/shapepipe/modules/pastecat_package/pastecat.py b/src/shapepipe/modules/pastecat_package/pastecat.py similarity index 100% rename from shapepipe/modules/pastecat_package/pastecat.py rename to src/shapepipe/modules/pastecat_package/pastecat.py diff --git a/shapepipe/modules/pastecat_runner.py b/src/shapepipe/modules/pastecat_runner.py similarity index 100% rename from shapepipe/modules/pastecat_runner.py rename to src/shapepipe/modules/pastecat_runner.py diff --git a/shapepipe/modules/psfex_interp_package/__init__.py b/src/shapepipe/modules/psfex_interp_package/__init__.py similarity index 100% rename from shapepipe/modules/psfex_interp_package/__init__.py rename to src/shapepipe/modules/psfex_interp_package/__init__.py diff --git a/shapepipe/modules/psfex_interp_package/psfex_interp.py b/src/shapepipe/modules/psfex_interp_package/psfex_interp.py similarity index 100% rename from shapepipe/modules/psfex_interp_package/psfex_interp.py rename to src/shapepipe/modules/psfex_interp_package/psfex_interp.py diff --git a/shapepipe/modules/psfex_interp_runner.py b/src/shapepipe/modules/psfex_interp_runner.py similarity index 100% rename from shapepipe/modules/psfex_interp_runner.py rename to src/shapepipe/modules/psfex_interp_runner.py diff --git a/shapepipe/modules/psfex_package/__init__.py b/src/shapepipe/modules/psfex_package/__init__.py similarity index 100% rename from shapepipe/modules/psfex_package/__init__.py rename to src/shapepipe/modules/psfex_package/__init__.py diff --git a/shapepipe/modules/psfex_package/psfex_script.py b/src/shapepipe/modules/psfex_package/psfex_script.py similarity index 100% rename from shapepipe/modules/psfex_package/psfex_script.py rename to src/shapepipe/modules/psfex_package/psfex_script.py diff --git a/shapepipe/modules/psfex_runner.py b/src/shapepipe/modules/psfex_runner.py similarity index 100% rename from shapepipe/modules/psfex_runner.py rename to src/shapepipe/modules/psfex_runner.py diff --git a/shapepipe/modules/python_example_package/__init__.py b/src/shapepipe/modules/python_example_package/__init__.py similarity index 100% rename from shapepipe/modules/python_example_package/__init__.py rename to src/shapepipe/modules/python_example_package/__init__.py diff --git a/shapepipe/modules/python_example_package/python_example.py b/src/shapepipe/modules/python_example_package/python_example.py similarity index 100% rename from shapepipe/modules/python_example_package/python_example.py rename to src/shapepipe/modules/python_example_package/python_example.py diff --git a/shapepipe/modules/python_example_runner.py b/src/shapepipe/modules/python_example_runner.py similarity index 100% rename from shapepipe/modules/python_example_runner.py rename to src/shapepipe/modules/python_example_runner.py diff --git a/shapepipe/modules/random_cat_package/__init__.py b/src/shapepipe/modules/random_cat_package/__init__.py similarity index 100% rename from shapepipe/modules/random_cat_package/__init__.py rename to src/shapepipe/modules/random_cat_package/__init__.py diff --git a/shapepipe/modules/random_cat_package/random_cat.py b/src/shapepipe/modules/random_cat_package/random_cat.py similarity index 100% rename from shapepipe/modules/random_cat_package/random_cat.py rename to src/shapepipe/modules/random_cat_package/random_cat.py diff --git a/shapepipe/modules/random_cat_runner.py b/src/shapepipe/modules/random_cat_runner.py similarity index 100% rename from shapepipe/modules/random_cat_runner.py rename to src/shapepipe/modules/random_cat_runner.py diff --git a/shapepipe/modules/serial_example_runner.py b/src/shapepipe/modules/serial_example_runner.py similarity index 100% rename from shapepipe/modules/serial_example_runner.py rename to src/shapepipe/modules/serial_example_runner.py diff --git a/shapepipe/modules/setools_package/__init__.py b/src/shapepipe/modules/setools_package/__init__.py similarity index 100% rename from shapepipe/modules/setools_package/__init__.py rename to src/shapepipe/modules/setools_package/__init__.py diff --git a/shapepipe/modules/setools_package/setools.py b/src/shapepipe/modules/setools_package/setools.py similarity index 100% rename from shapepipe/modules/setools_package/setools.py rename to src/shapepipe/modules/setools_package/setools.py diff --git a/shapepipe/modules/setools_runner.py b/src/shapepipe/modules/setools_runner.py similarity index 100% rename from shapepipe/modules/setools_runner.py rename to src/shapepipe/modules/setools_runner.py diff --git a/shapepipe/modules/sextractor_package/__init__.py b/src/shapepipe/modules/sextractor_package/__init__.py similarity index 100% rename from shapepipe/modules/sextractor_package/__init__.py rename to src/shapepipe/modules/sextractor_package/__init__.py diff --git a/shapepipe/modules/sextractor_package/sextractor_script.py b/src/shapepipe/modules/sextractor_package/sextractor_script.py similarity index 100% rename from shapepipe/modules/sextractor_package/sextractor_script.py rename to src/shapepipe/modules/sextractor_package/sextractor_script.py diff --git a/shapepipe/modules/sextractor_runner.py b/src/shapepipe/modules/sextractor_runner.py similarity index 100% rename from shapepipe/modules/sextractor_runner.py rename to src/shapepipe/modules/sextractor_runner.py diff --git a/shapepipe/modules/split_exp_package/__init__.py b/src/shapepipe/modules/split_exp_package/__init__.py similarity index 100% rename from shapepipe/modules/split_exp_package/__init__.py rename to src/shapepipe/modules/split_exp_package/__init__.py diff --git a/shapepipe/modules/split_exp_package/split_exp.py b/src/shapepipe/modules/split_exp_package/split_exp.py similarity index 100% rename from shapepipe/modules/split_exp_package/split_exp.py rename to src/shapepipe/modules/split_exp_package/split_exp.py diff --git a/shapepipe/modules/split_exp_runner.py b/src/shapepipe/modules/split_exp_runner.py similarity index 100% rename from shapepipe/modules/split_exp_runner.py rename to src/shapepipe/modules/split_exp_runner.py diff --git a/shapepipe/modules/spread_model_package/__init__.py b/src/shapepipe/modules/spread_model_package/__init__.py similarity index 100% rename from shapepipe/modules/spread_model_package/__init__.py rename to src/shapepipe/modules/spread_model_package/__init__.py diff --git a/shapepipe/modules/spread_model_package/spread_model.py b/src/shapepipe/modules/spread_model_package/spread_model.py similarity index 100% rename from shapepipe/modules/spread_model_package/spread_model.py rename to src/shapepipe/modules/spread_model_package/spread_model.py diff --git a/shapepipe/modules/spread_model_runner.py b/src/shapepipe/modules/spread_model_runner.py similarity index 100% rename from shapepipe/modules/spread_model_runner.py rename to src/shapepipe/modules/spread_model_runner.py diff --git a/shapepipe/modules/uncompress_fits_package/__init__.py b/src/shapepipe/modules/uncompress_fits_package/__init__.py similarity index 100% rename from shapepipe/modules/uncompress_fits_package/__init__.py rename to src/shapepipe/modules/uncompress_fits_package/__init__.py diff --git a/shapepipe/modules/uncompress_fits_package/uncompress_fits.py b/src/shapepipe/modules/uncompress_fits_package/uncompress_fits.py similarity index 100% rename from shapepipe/modules/uncompress_fits_package/uncompress_fits.py rename to src/shapepipe/modules/uncompress_fits_package/uncompress_fits.py diff --git a/shapepipe/modules/uncompress_fits_runner.py b/src/shapepipe/modules/uncompress_fits_runner.py similarity index 100% rename from shapepipe/modules/uncompress_fits_runner.py rename to src/shapepipe/modules/uncompress_fits_runner.py diff --git a/shapepipe/modules/vignetmaker_package/__init__.py b/src/shapepipe/modules/vignetmaker_package/__init__.py similarity index 100% rename from shapepipe/modules/vignetmaker_package/__init__.py rename to src/shapepipe/modules/vignetmaker_package/__init__.py diff --git a/shapepipe/modules/vignetmaker_package/vignetmaker.py b/src/shapepipe/modules/vignetmaker_package/vignetmaker.py similarity index 100% rename from shapepipe/modules/vignetmaker_package/vignetmaker.py rename to src/shapepipe/modules/vignetmaker_package/vignetmaker.py diff --git a/shapepipe/modules/vignetmaker_runner.py b/src/shapepipe/modules/vignetmaker_runner.py similarity index 100% rename from shapepipe/modules/vignetmaker_runner.py rename to src/shapepipe/modules/vignetmaker_runner.py diff --git a/shapepipe/pipeline/__init__.py b/src/shapepipe/pipeline/__init__.py similarity index 100% rename from shapepipe/pipeline/__init__.py rename to src/shapepipe/pipeline/__init__.py diff --git a/shapepipe/pipeline/args.py b/src/shapepipe/pipeline/args.py similarity index 97% rename from shapepipe/pipeline/args.py rename to src/shapepipe/pipeline/args.py index e06e44ef9..4bd574180 100644 --- a/shapepipe/pipeline/args.py +++ b/src/shapepipe/pipeline/args.py @@ -8,7 +8,8 @@ import argparse as ap -from shapepipe.info import __version__, shapepipe_logo +from shapepipe import __version__ +from shapepipe.info import shapepipe_logo from shapepipe.modules import __module_list__ diff --git a/shapepipe/pipeline/config.py b/src/shapepipe/pipeline/config.py similarity index 100% rename from shapepipe/pipeline/config.py rename to src/shapepipe/pipeline/config.py diff --git a/shapepipe/pipeline/dependency_handler.py b/src/shapepipe/pipeline/dependency_handler.py similarity index 100% rename from shapepipe/pipeline/dependency_handler.py rename to src/shapepipe/pipeline/dependency_handler.py diff --git a/shapepipe/pipeline/execute.py b/src/shapepipe/pipeline/execute.py similarity index 100% rename from shapepipe/pipeline/execute.py rename to src/shapepipe/pipeline/execute.py diff --git a/shapepipe/pipeline/file_handler.py b/src/shapepipe/pipeline/file_handler.py similarity index 100% rename from shapepipe/pipeline/file_handler.py rename to src/shapepipe/pipeline/file_handler.py diff --git a/shapepipe/pipeline/file_io.py b/src/shapepipe/pipeline/file_io.py similarity index 100% rename from shapepipe/pipeline/file_io.py rename to src/shapepipe/pipeline/file_io.py diff --git a/shapepipe/pipeline/job_handler.py b/src/shapepipe/pipeline/job_handler.py similarity index 100% rename from shapepipe/pipeline/job_handler.py rename to src/shapepipe/pipeline/job_handler.py diff --git a/shapepipe/pipeline/mpi_run.py b/src/shapepipe/pipeline/mpi_run.py similarity index 100% rename from shapepipe/pipeline/mpi_run.py rename to src/shapepipe/pipeline/mpi_run.py diff --git a/shapepipe/pipeline/run_log.py b/src/shapepipe/pipeline/run_log.py similarity index 100% rename from shapepipe/pipeline/run_log.py rename to src/shapepipe/pipeline/run_log.py diff --git a/shapepipe/pipeline/shared.py b/src/shapepipe/pipeline/shared.py similarity index 100% rename from shapepipe/pipeline/shared.py rename to src/shapepipe/pipeline/shared.py diff --git a/shapepipe/pipeline/str_handler.py b/src/shapepipe/pipeline/str_handler.py similarity index 100% rename from shapepipe/pipeline/str_handler.py rename to src/shapepipe/pipeline/str_handler.py diff --git a/shapepipe/pipeline/timeout.py b/src/shapepipe/pipeline/timeout.py similarity index 100% rename from shapepipe/pipeline/timeout.py rename to src/shapepipe/pipeline/timeout.py diff --git a/shapepipe/pipeline/worker_handler.py b/src/shapepipe/pipeline/worker_handler.py similarity index 100% rename from shapepipe/pipeline/worker_handler.py rename to src/shapepipe/pipeline/worker_handler.py diff --git a/shapepipe/run.py b/src/shapepipe/run.py similarity index 98% rename from shapepipe/run.py rename to src/shapepipe/run.py index 2443eab50..c494df836 100644 --- a/shapepipe/run.py +++ b/src/shapepipe/run.py @@ -8,12 +8,13 @@ import sys from datetime import datetime +from importlib.metadata import requires from joblib import cpu_count from modopt.interface.errors import catch_error from modopt.interface.log import close_log, set_up_log -from shapepipe.info import __installs__, line, shapepipe_logo +from shapepipe.info import line, shapepipe_logo from shapepipe.pipeline.args import create_arg_parser from shapepipe.pipeline.config import create_config_parser from shapepipe.pipeline.dependency_handler import DependencyHandler @@ -172,9 +173,9 @@ def _check_dependencies(self): Check that all pipeline dependencies have been installed. """ - module_dep = self._get_module_depends('depends') + __installs__ + module_dep = self._get_module_depends('depends') + [dep.split()[0] for dep in requires("ShapePipe") if "extra ==" not in dep] module_exe = self._get_module_depends('executes') - + module_dep += ['mpi4py'] if import_mpi else module_dep dh = DependencyHandler(module_dep, module_exe) diff --git a/scripts/python/shapepipe_run.py b/src/shapepipe/shapepipe_run.py similarity index 100% rename from scripts/python/shapepipe_run.py rename to src/shapepipe/shapepipe_run.py diff --git a/shapepipe/tests/test_get_images.py b/src/shapepipe/tests/test_get_images.py similarity index 100% rename from shapepipe/tests/test_get_images.py rename to src/shapepipe/tests/test_get_images.py diff --git a/shapepipe/tests/test_pipeline.py b/src/shapepipe/tests/test_pipeline.py similarity index 100% rename from shapepipe/tests/test_pipeline.py rename to src/shapepipe/tests/test_pipeline.py diff --git a/shapepipe/tests/test_utilities.py b/src/shapepipe/tests/test_utilities.py similarity index 100% rename from shapepipe/tests/test_utilities.py rename to src/shapepipe/tests/test_utilities.py diff --git a/shapepipe/utilities/__init__.py b/src/shapepipe/utilities/__init__.py similarity index 100% rename from shapepipe/utilities/__init__.py rename to src/shapepipe/utilities/__init__.py diff --git a/shapepipe/utilities/cfis.py b/src/shapepipe/utilities/cfis.py similarity index 100% rename from shapepipe/utilities/cfis.py rename to src/shapepipe/utilities/cfis.py diff --git a/shapepipe/utilities/file_system.py b/src/shapepipe/utilities/file_system.py similarity index 100% rename from shapepipe/utilities/file_system.py rename to src/shapepipe/utilities/file_system.py diff --git a/shapepipe/utilities/galaxy.py b/src/shapepipe/utilities/galaxy.py similarity index 100% rename from shapepipe/utilities/galaxy.py rename to src/shapepipe/utilities/galaxy.py From bb79e707a3a829ca8d09e6bde7c5f81db82b7c8a Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sun, 13 Oct 2024 11:05:10 +0200 Subject: [PATCH 08/25] set checkout action version --- .github/workflows/ci-dev.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index 1ce5c0070..4ba9d179b 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -18,7 +18,7 @@ jobs: python-version: [3.11] steps: - - uses: actions/checkout + - uses: actions/checkout@4 - name: Install Linux dependencies if: runner.os == 'Linux' @@ -32,7 +32,7 @@ jobs: brew install bigmac libomp - name: Set up conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.python-version }} From c680f88617c914c2ee8a64725936d2447ce7923e Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sun, 13 Oct 2024 11:06:12 +0200 Subject: [PATCH 09/25] fix checkout action version --- .github/workflows/ci-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index 4ba9d179b..0a303072c 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -18,7 +18,7 @@ jobs: python-version: [3.11] steps: - - uses: actions/checkout@4 + - uses: actions/checkout@v4 - name: Install Linux dependencies if: runner.os == 'Linux' From fe3347aeb8baa25ac4923af1b2ddaf8c9626c60e Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sun, 13 Oct 2024 13:15:17 +0200 Subject: [PATCH 10/25] source Source Extractor from Conda-Forge --- environment-dev.yml | 2 ++ install_shapepipe | 70 ++++----------------------------------------- 2 files changed, 8 insertions(+), 64 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 95f8db62b..4c509658e 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -11,6 +11,7 @@ dependencies: - python=3.11 - pip>=21.2.4 - astropy>=5.0 + - astromatic-source-extractor>=2.25.0 - autoconf>=2.71 - automake>=1.16 - cmake>=3.27 @@ -19,6 +20,7 @@ dependencies: - libtool>=2.4 - matplotlib>=3.8 - numba>=0.57 + - numpy>=1.22 - pandas>=2.1 - pyqt5-sip>=12.12 - pyqtgraph>=0.13 diff --git a/install_shapepipe b/install_shapepipe index 4c3f512c6..bd0a2ce72 100755 --- a/install_shapepipe +++ b/install_shapepipe @@ -10,8 +10,8 @@ export LC_CTYPE=en_US.utf8 # ############################################################################## -version="1.6" -last_update="08/03/22" +version="1.7" +last_update="13/10/24" ############################################################################## # EXTERNAL PACKAGES AND SCRIPTS @@ -25,12 +25,8 @@ libpng_ver="1.6.37" mpi4py_ver="3.1.3" openblas_ver="0.3.18" -# SExtractor Package -sex_url="https://github.com/astromatic/sextractor/archive/2.25.0.tar.gz" -sex_tar="2.25.0.tar.gz" -sex_dir="sextractor-2.25.0" -sex_bin="sex" -sex_version=TRUE +# Source Extractor +se_bin="sex" # PSFEx Package psfex_url="https://github.com/astromatic/psfex/archive/3.21.1.tar.gz" @@ -86,7 +82,6 @@ USE_CONDA_SOURCE=FALSE BUILD_ENV=TRUE DEVELOP=FALSE VOS=FALSE -INSTALL_SEX=TRUE INSTALL_PSFEX=TRUE INSTALL_WW=TRUE INSTALL_CDSCLIENT=TRUE @@ -107,36 +102,6 @@ download_package() { rm $2 } -# Function to build SExtractor with OpenBLAS -build_sex_blas() { - astromatic_setup - cd $BUILD_DIR/$1 - ./autogen.sh - ./configure --prefix=$CONDA_PREFIX --enable-openblas --with-fftw-libdir=$FFTW_LIB \ - --with-fftw-incdir=$FFTW_INC --with-openblas-libdir=$BLAS_LIB \ - --with-openblas-incdir=$BLAS_INC --quiet - make -j --quiet - make install -} - -# Function to build SExtractor with ATLAS -build_sex_atlas() { - astromatic_setup - cd $BUILD_DIR/$1 - ./autogen.sh - if [ "$ATLAS_SEARCH" == TRUE ] - then - ./configure --prefix=$CONDA_PREFIX --with-fftw-libdir=$FFTW_LIB \ - --with-fftw-incdir=$FFTW_INC --quiet - else - ./configure --prefix=$CONDA_PREFIX --with-fftw-libdir=$FFTW_LIB \ - --with-fftw-incdir=$FFTW_INC --with-atlas-libdir=$ATLAS_LIB \ - --with-atlas-incdir=$ATLAS_INC --quiet - fi - make -j --quiet - make install -} - # Function to build PSFEx with OpenBLAS build_psfex_blas() { astromatic_setup @@ -293,7 +258,7 @@ start() { echo $line echo 'ShapePipe Installation Script' echo '' - echo 'Author: Samuel Farrens' + echo 'Authors: Samuel Farrens and Martin Kilbinger' echo 'Version:' $version echo 'Last Update:' $last_update echo $line @@ -302,7 +267,7 @@ start() { # Function to report package versions packages() { - check_binary2 $sex_bin "SExtractor" $sex_version $INSTALL_SEX + check_binary2 $se_bin "Source Extractor" TRUE TRUE check_binary2 $psfex_bin "PSFEx" $psfex_version $INSTALL_PSFEX check_binary2 $ww_bin "WeightWatcher" $ww_version $INSTALL_WW check_binary2 $cdsclient_bin "CDSclient" $cdsclient_version $INSTALL_CDSCLIENT @@ -344,7 +309,6 @@ package_report() { then activate_shapepipe fi - INSTALL_SEX=FALSE INSTALL_PSFEX=FALSE INSTALL_WW=FALSE INSTALL_CDSCLIENT=FALSE @@ -395,7 +359,6 @@ setup() { check_status 'Operating System:' $SYSOS check_status 'Conda Version:' $CONDA_VERSION check_status 'Build Conda Environment:' $BUILD_ENV $ENV_NAME - check_status 'Install SExtractor:' $INSTALL_SEX check_status 'Install PSFEx:' $INSTALL_PSFEX check_status 'Install WeightWatcher:' $INSTALL_WW check_status 'Install CDSclient:' $INSTALL_CDSCLIENT @@ -484,7 +447,6 @@ Options:\n \t--develop\t option to install additional development packages\n \t--vos\t\t option to install virtual observatory software\n \t--no-env\t do not build Conda environment\n -\t--no-sex\t do not build SExtractor\n \t--no-psfex\t do not build PSFEx\n \t--no-ww\t\t do not build WeightWatcher\n \t--no-cds\t do not build CDSclient\n @@ -562,10 +524,6 @@ case $i in BUILD_ENV=FALSE shift ;; - --no-sex) - INSTALL_SEX=FALSE - shift - ;; --no-psfex) INSTALL_PSFEX=FALSE shift @@ -615,7 +573,6 @@ case $i in shift ;; --no-exe) - INSTALL_SEX=FALSE INSTALL_PSFEX=FALSE INSTALL_WW=FALSE INSTALL_CDSCLIENT=FALSE @@ -749,21 +706,6 @@ fi # BUILD EXTERNAL EXECUTABLES ############################################################################## -# Build SExtractor -if [ "$INSTALL_SEX" == TRUE ] && check_binary $sex_bin -then - report_progress 'SExtractor' - conda install -n $ENV_NAME -c conda-forge "fftw==${fftw_ver}" -y - download_package $sex_url $sex_tar - if [ "$use_atlas" == TRUE ] - then - build_sex_atlas $sex_dir - else - conda install -n $ENV_NAME -c conda-forge "openblas==${openblas_ver}" -y - build_sex_blas $sex_dir - fi -fi - # Build PSFEx if [ "$INSTALL_PSFEX" == TRUE ] && check_binary $psfex_bin then From a562c77f7d7e6fb36f32ed4113da87d4b1d804ca Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sun, 13 Oct 2024 13:15:48 +0200 Subject: [PATCH 11/25] test Apple Silicon workflow --- .github/workflows/ci-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index 0a303072c..cdac3c422 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest-xlarge] python-version: [3.11] steps: From 0a61f526ba25f8744b90aeea969f307a74e5872d Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sun, 13 Oct 2024 13:24:03 +0200 Subject: [PATCH 12/25] revert macOS runner --- .github/workflows/ci-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index cdac3c422..0a303072c 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest-xlarge] + os: [ubuntu-latest, macos-latest] python-version: [3.11] steps: From 43ce60e57326794b324bf8f93cbb65feea8016f3 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sun, 13 Oct 2024 13:58:17 +0200 Subject: [PATCH 13/25] add gfortran to macOS deps --- .github/workflows/ci-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index 0a303072c..c37ecf28b 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -29,7 +29,7 @@ jobs: if: runner.os == 'macOS' run: | brew tap sfarrens/sf - brew install bigmac libomp + brew install bigmac gfortran libomp - name: Set up conda uses: conda-incubator/setup-miniconda@v3 From 354aa184691ded0004d6c94eae6f18e1f5a29bc0 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sun, 13 Oct 2024 14:06:18 +0200 Subject: [PATCH 14/25] test reinstall of gcc to find gfortran --- .github/workflows/ci-dev.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index c37ecf28b..1c08b90f6 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -29,7 +29,8 @@ jobs: if: runner.os == 'macOS' run: | brew tap sfarrens/sf - brew install bigmac gfortran libomp + brew install bigmac libomp + brew reinstall gcc - name: Set up conda uses: conda-incubator/setup-miniconda@v3 From 0879e3e86aa5eddb1d92c102a7142d5f8cc95244 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sun, 13 Oct 2024 14:39:19 +0200 Subject: [PATCH 15/25] test without symlinks --- install_shapepipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_shapepipe b/install_shapepipe index bd0a2ce72..79f571ff7 100755 --- a/install_shapepipe +++ b/install_shapepipe @@ -748,7 +748,7 @@ cd $PIPE_DIR python -m pip install . # Create symbolic links to scripts -build_symlinks +# build_symlinks finish From 0db1f605d564c0a25f23ea8541c6120207ca9ea1 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sun, 13 Oct 2024 16:08:15 +0200 Subject: [PATCH 16/25] update symlink script and dev requirements --- develop.txt | 6 --- docs/requirements.txt | 5 --- environment-dev.yml | 1 + install_shapepipe | 95 +++++++++++++++++++++++++------------------ pyproject.toml | 18 +++++++- 5 files changed, 74 insertions(+), 51 deletions(-) delete mode 100644 develop.txt delete mode 100644 docs/requirements.txt diff --git a/develop.txt b/develop.txt deleted file mode 100644 index 35596ad9c..000000000 --- a/develop.txt +++ /dev/null @@ -1,6 +0,0 @@ -isort -numpydoc -pytest -pytest-cov -pytest-pycodestyle -pytest-pydocstyle diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 8aa90168c..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -myst-parser==0.17.0 -numpydoc==1.2 -sphinx==4.4.0 -sphinxcontrib-bibtex==2.4.1 -sphinx-book-theme==0.2.0 diff --git a/environment-dev.yml b/environment-dev.yml index 4c509658e..eb268f74d 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -29,6 +29,7 @@ dependencies: - termcolor>=2.3 - tqdm>=4.66 - treecorr>=4.3 + - wget - pip: - cs_util>=0.0.5 - mccd>=1.2.3 diff --git a/install_shapepipe b/install_shapepipe index 79f571ff7..a72e25fbb 100755 --- a/install_shapepipe +++ b/install_shapepipe @@ -22,22 +22,22 @@ last_update="13/10/24" # Conda package versions fftw_ver="3.3.10" libpng_ver="1.6.37" -mpi4py_ver="3.1.3" +mpi4py_ver="4.0.1" openblas_ver="0.3.18" # Source Extractor se_bin="sex" # PSFEx Package -psfex_url="https://github.com/astromatic/psfex/archive/3.21.1.tar.gz" -psfex_tar="3.21.1.tar.gz" -psfex_dir="psfex-3.21.1" +psfex_url="https://github.com/astromatic/psfex/archive/3.24.1.tar.gz" +psfex_tar="3.24.1.tar.gz" +psfex_dir="psfex-3.24.1" psfex_bin="psfex" psfex_version=TRUE # WeightWatcher Package -ww_url="http://snapshot.debian.org/archive/debian/20171211T160522Z/pool/main/w/weightwatcher/weightwatcher_1.12.orig.tar.gz" -ww_tar="weightwatcher_1.12.orig.tar.gz" +ww_url="https://github.com/astromatic/weightwatcher/archive/refs/tags/1.12.tar.gz" +ww_tar="1.12.tar.gz" ww_dir="weightwatcher-1.12" ww_bin="ww" ww_version=TRUE @@ -56,13 +56,6 @@ mpi_dir="mpi4py-${mpi4py_ver}" mpi_bin="mpiexec" mpi_version=TRUE -# Allowed script extensions -script_ext=( - .py - .sh - .bash -) - # Divider line line="########################################################################" @@ -390,7 +383,7 @@ uninstall() { response=${response:-n} if [ $response == "y" ] then - rm -r build dist *.egg-info .eggs + rm -r build dist src/*.egg-info .eggs deactivate_shapepipe conda remove -n $ENV_NAME --all -y echo "Finished uninstalling $ENV_NAME" @@ -419,13 +412,27 @@ deactivate_shapepipe() { eval $CONDA_DEACTIVATE } -# Build symbolic links for scripts build_symlinks() { - for ext in ${script_ext[@]}; do - for script in $CONDA_PREFIX/bin/*$ext; do - link_name=`basename $script $ext` - ln -s $script $CONDA_PREFIX/bin/$link_name - done + echo "Creating symbolic links for scripts" + script_dirs=("scripts/python" "scripts/sh") + for dir in "${script_dirs[@]}"; do + for script in "$dir"/*; do + if [[ "$script" == *".sh" ]]; then + base_name=$(basename "$script" ".sh") + elif [[ "$script" == *".bash" ]]; then + base_name=$(basename "$script" ".bash") + elif [[ "$script" == *".py" ]]; then + base_name=$(basename "$script" ".py") + else + echo -ne "$2: ${YELLOW}WARNING:${NC}\n" + echo "Unknown script extension for $script." + continue + fi + # Create symbolic link inside $CONDA_PREFIX/bin without extension + ln -sf "$(pwd)/$script" "$CONDA_PREFIX/bin/$base_name" + # Make the symbolic link executable + chmod +x "$CONDA_PREFIX/bin/$base_name" + done done } @@ -646,18 +653,6 @@ if [ -z "$FFTW_INC" ]; then FFTW_INC=$CONDA_PREFIX/include; fi if [ -z "$BLAS_LIB" ]; then BLAS_LIB=$CONDA_PREFIX/lib; fi if [ -z "$BLAS_INC" ]; then BLAS_INC=$CONDA_PREFIX/include; fi -############################################################################## -# INSTALL DEVELOPER PACKAGES -############################################################################## - -# Install development packages -if [ "$DEVELOP" == TRUE ] -then - report_progress 'Developer Packages' - python -m pip install -r develop.txt - python -m pip install -r docs/requirements.txt -fi - ############################################################################## # INSTALL THE VOS LIBRARY ############################################################################## @@ -677,7 +672,7 @@ fi if [ "$SYSOS" == "macOS" ] then report_progress 'macOS Requirements' - conda install -n $ENV_NAME wget -y + echo "Setting C compiler flags" export C_INCLUDE_PATH=$CONDA_PREFIX/include export CFLAGS="-Wl,-rpath,$CONDA_PREFIX/lib" export CPPFLAGS="-Wno-everything" @@ -695,7 +690,12 @@ then report_progress 'MPI' if [ -z "$MPI_ROOT" ] then - conda install -n $ENV_NAME -c conda-forge "mpi4py==${mpi4py_ver}" -y + if [ "$ENV_DEV" == TRUE ] + then + conda install -n $ENV_NAME -c conda-forge "mpi4py" -y + else + conda install -n $ENV_NAME -c conda-forge "mpi4py==${mpi4py_ver}" -y + fi else download_package $mpi_url $mpi_tar build_mpi $mpi_dir @@ -710,14 +710,23 @@ fi if [ "$INSTALL_PSFEX" == TRUE ] && check_binary $psfex_bin then report_progress 'PSFEx' - conda install "libpng==${libpng_ver}" -y - conda install -n $ENV_NAME -c conda-forge "fftw==${fftw_ver}" -y + if [ "$ENV_DEV" == TRUE ] + then + conda install -n $ENV_NAME -c conda-forge libpng fftw -y + else + conda install -n $ENV_NAME -c conda-forge "libpng==${libpng_ver}" "fftw==${fftw_ver}" -y + fi download_package $psfex_url $psfex_tar if [ "$use_atlas" == TRUE ] then build_psfex_atlas $psfex_dir else - conda install -n $ENV_NAME -c conda-forge "openblas==${openblas_ver}" -y + if [ "$ENV_DEV" == TRUE ] + then + conda install -n $ENV_NAME -c conda-forge "openblas" -y + else + conda install -n $ENV_NAME -c conda-forge "openblas==${openblas_ver}" -y + fi build_psfex_blas $psfex_dir fi fi @@ -745,10 +754,18 @@ fi # Install ShapePipe package report_progress 'ShapePipe' cd $PIPE_DIR -python -m pip install . + +if [ "$DEVELOP" == "TRUE" ] || [ "$ENV_DEV" == "TRUE" ] +then + echo "Installing ShapePipe package with developer tools" + python -m pip install ".[dev]" +else + echo "Installing ShapePipe package" + python -m pip install . +fi # Create symbolic links to scripts -# build_symlinks +build_symlinks finish diff --git a/pyproject.toml b/pyproject.toml index b038eb1cd..c9003c729 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,28 @@ dependencies = [ ] [project.optional-dependencies] -testing = [ +doc = [ + "myst-parser", + "numpydoc", + "sphinx", + "sphinxcontrib-bibtex", + "sphinx-book-theme" +] +lint = [ + "black", + "isort" +] +release = [ + "build", + "twine", +] +test = [ "pytest", "pytest-cov", "pytest-pycodestyle", "pytest-pydocstyle" ] +dev = ["shapepipe[doc,lint,release,test]"] [project.scripts] shapepipe_run = "shapepipe.shapepipe_run:main" From 18643a6ed4423901c8641f8bdf2d77f252c9cb16 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sun, 13 Oct 2024 16:28:55 +0200 Subject: [PATCH 17/25] update LDFLAGS and add license --- LICENSE | 113 ++++++++++++++++++++++++++++++++++++++++++++++ install_shapepipe | 2 + pyproject.toml | 4 ++ 3 files changed, 119 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..2b83262ae --- /dev/null +++ b/LICENSE @@ -0,0 +1,113 @@ +# LICENCES + +Le code source de ce dépôt est publié sous [licence MIT](#licence-mit). + +Sauf mention de propriété intellectuelle détenue par des tiers (notamment +un crédit sur certaines images), les contenus de ce dépôt sont publiés sous [licence Ouverte 2.0](#licence-ouverte-20open-licence-20). + +La marque d'État est à usage exclusif des acteurs de la sphère +étatique. En particulier, la typographie Marianne© est protégée par +le droit d'auteur. [Lire les explications sur le site de la marque +d'État.](https://www.gouvernement.fr/charte/charte-graphique-les-fondamentaux/la-typographie) + +## LICENCE MIT + +Copyright (c) 2021 DINUM + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +## LICENCE OUVERTE 2.0/OPEN LICENCE 2.0 + +### Réutilisation de l’« Information » sous cette licence + +Le « Concédant » concède au « Réutilisateur » un droit non exclusif et gratuit de libre « Réutilisation » de l’« Information » objet de la présente licence, à des fins commerciales ou non, dans le monde entier et pour une durée illimitée, dans les conditions exprimées ci-dessous. + +**Le « Réutilisateur » est libre de réutiliser l’« Information » :** + +- de la communiquer, la reproduire, la copier ; +- de l’adapter, la modifier, l’extraire et la transformer, notamment pour créer des « Informations dérivées » ; +- de la diffuser, la redistribuer, la publier et la transmettre, de l’exploiter à titre commercial, par exemple en la combinant avec d’autres informations, ou en l’incluant dans votre propre produit ou application. + +**Sous réserve de :** + +- mentionner la paternité de l’«Information» : sa source (a minima le nom du « Concédant ») et la date de la dernière mise à jour de l’« Information » réutilisée. + +Le « Réutilisateur » peut notamment s’ acquitter de cette condition en indiquant l’adresse (URL) renvoyant vers « l’Information » et assurant une mention effective de sa paternité. + +**Par exemple :** + +Dans le cas d’une réutilisation de la base SIRENE de l’INSEE, mentionner l’URL du « Concédant » : www.insee.fr + la date de dernière mise à jour de l’Information réutilisée. + +Cette mention de paternité ne doit ni conférer un caractère officiel à la « Réutilisation » de l’« Information », ni suggérer une quelconque reconnaissance ou caution par le « Concédant », ou par toute autre entité publique, du « Réutilisateur » ou de sa « Réutilisation ». + +### Données à caractère personnel + +L’« Information » mise à disposition peut contenir des « Données à caractère personnel » pouvant faire l’objet d’une « Réutilisation ». Alors, le « Concédant » informe le « Réutilisateur » (par tous moyens) de leur présence, l’ « Information » peut être librement réutilisée, sans faire obstacle aux libertés accordées par la présente licence, à condition de respecter le cadre légal relatif à la protection des données à caractère personnel. + +### Droits de propriété intellectuelle + +Il est garanti au « Réutilisateur » que l’ « Information » ne contient pas de « Droits de propriété intellectuelle » appartenant à des tiers qui pourraient faire obstacle aux libertés qui lui sont accordées par la présente licence. + +Les éventuels « Droits de propriété intellectuelle » détenus par le « Concédant » sur l’ « Information » ne font pas obstacle aux libertés qui sont accordées par la présente licence. Lorsque le « Concédant » détient des « Droits de propriété intellectuelle » » sur l’ « Information », il les cède au « Réutilisateur » de façon non exclusive, à titre gracieux, pour le monde entier, pour toute la durée des « Droits de propriété intellectuelle », et le « Réutilisateur » peut en faire tout usage conformément aux libertés et aux conditions définies par la présente licence. + +### Responsabilité + +L’ «Information» est mise à disposition telle que produite ou reçue, sans autre garantie expresse ou tacite qui n’est pas prévue par la présente licence. L’absence de défauts ou d’erreurs éventuellement contenues dans l’ «Information», comme la fourniture continue de l’ « Information » n’est pas garantie par le «Concédant». Il ne peut être tenu pour responsable de toute perte, préjudice ou dommage de quelque sorte causé à des tiers du fait de la « Réutilisation ». + +Le « Réutilisateur » est seul responsable de la « Réutilisation » de l’« Information ». + +La « Réutilisation » ne doit pas induire en erreur des tiers quant au contenu de l’« Information », sa source et sa date de mise à jour. + +### Droit applicable + +La présente licence est régie par le droit français. + +#### Compatibilité de la présente licence + +Elle a été conçue pour être compatible avec toute licence libre qui exige _a minima_ la mention de paternité. Elle est notamment compatible avec la version antérieure de la présente licence ainsi qu’avec les licences « Open Government Licence » (OGL) du Royaume-Uni, « Creative Commons Attribution » (CC-BY) de Creative Commons et « Open Data Commons Attribution » (ODC-BY) de l’Open Knowledge Foundation. + +### Définitions + +Sont considérés, au sens de la présente licence comme : + +- Le « **Concédant** » : toute personne concédant un droit de « Réutilisation » sur l’« Information » dans les libertés et les conditions prévues par la présente licence. +- L’« **Information** » : + - toute information publique figurant dans des documents communiqués ou publiés par une administration mentionnée au premier alinéa de l’article L.300-2 du CRPA ; + - toute information mise à disposition par toute personne selon les termes et conditions de la présente licence. +- La « **Réutilisation** » : l’utilisation de l’« Information » à d’autres fins que celles pour lesquelles elle a été produite ou reçue. +- Le « **Réutilisateur** » : toute personne qui réutilise les « Informations » conformément aux conditions de la présente licence. +- Des « **Données à caractère personnel** » : toute information se rapportant à une personne physique identifiée ou identifiable, pouvant être identifiée directement ou indirectement. Leur « Réutilisation » est subordonnée au respect du cadre juridique en vigueur. +- Une « **Information dérivée** » : toute nouvelle donnée ou information créées directement à partir de l’« Information » ou à partir d’une combinaison de l’ « Information » et d’autres données ou informations non soumises à cette licence. +- Les « **Droits de propriété intellectuelle** » : tous droits identifiés comme tels par le Code de la propriété intellectuelle (droit d’auteur, droits voisins au droit d’auteur, droit sui generis des producteurs de bases de données). + +### À propos de cette licence + +La présente licence a vocation à être utilisée par les administrations pour la réutilisation de leurs informations publiques. Elle peut également être utilisée par toute personne souhaitant mettre à disposition de l’« Information » dans les conditions définies par la présente licence + +La France est dotée d’un cadre juridique global visant à une diffusion spontanée par les administrations de leurs informations publiques afin d’en permettre la plus large réutilisation. + +Le droit de la « Réutilisation » de l’« Information » des administrations est régi par le code des relations entre le public et l’administration (CRPA) et, le cas échéant, le code du patrimoine (livre II relatif aux archives). + +Cette licence facilite la réutilisation libre et gratuite des informations publiques et figure parmi les licences qui peuvent être utilisées par l’administration en vertu du décret pris en application de l’article L.323-2 du CRPA. + +Etalab est la mission chargée, sous l’autorité du Premier ministre, d’ouvrir le plus grand nombre de données publiques des administrations de l’État et de ses établissements publics. Elle a réalisé la Licence Ouverte pour faciliter la réutilisation libre et gratuite de ces informations publiques, telles que définies par l’article L321-1 du CRPA. + +Cette licence est une version 2.0 de la Licence Ouverte. + +Etalab se réserve la faculté de proposer de nouvelles versions de la Licence Ouverte. Cependant, les « Réutilisateurs » pourront continuer à réutiliser les informations disponibles sous cette licence s’ils le souhaitent. diff --git a/install_shapepipe b/install_shapepipe index a72e25fbb..c88bdffc0 100755 --- a/install_shapepipe +++ b/install_shapepipe @@ -363,6 +363,8 @@ setup() { # Function to show the astromatic setup astromatic_setup() { + export LDFLAGS="-lpthread" + echo 'LDFLAGS:' $LDFLAGS echo 'FFTW Library Path:' $FFTW_LIB echo 'FFTW Include Path:' $FFTW_INC if [ "$use_atlas" == TRUE ] diff --git a/pyproject.toml b/pyproject.toml index c9003c729..ab87fac6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,17 +2,21 @@ name = "shapepipe" version = "1.0.2" description = "Galaxy shape measurement pipeline." +readme = "README.rst" authors = [ { name = "Samuel Farrens", email = "samuel.farrens@cea.fr" }, { name = "Axel Guinot", email = "axel.guinot@cea.fr" }, { name = "Martin Kilbinger", email = "martin.kilbinger@cea.fr" } ] +license = { "file" = "LICENSE" } +requires-python = ">=3.11" dependencies = [ "joblib>=0.13", "modopt>=1.2", "numpy>=1.14" ] + [project.optional-dependencies] doc = [ "myst-parser", From 39e7af46db24d63eda95cd798c6199f47c60872f Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sun, 13 Oct 2024 16:43:51 +0200 Subject: [PATCH 18/25] test CXXFLAGS='-fcommon' for ww --- install_shapepipe | 1 + 1 file changed, 1 insertion(+) diff --git a/install_shapepipe b/install_shapepipe index c88bdffc0..88ec7a2a6 100755 --- a/install_shapepipe +++ b/install_shapepipe @@ -738,6 +738,7 @@ if [ "$INSTALL_WW" == TRUE ] && check_binary $ww_bin then report_progress 'WeightWatcher' download_package $ww_url $ww_tar + export CXXFLAGS="-fcommon" build_standard $ww_dir fi From 16ab509814b9e9cdc2d54a36f4f248d43b65eff5 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Sat, 19 Oct 2024 14:34:09 +0200 Subject: [PATCH 19/25] test Ubuntu with gcc-9 --- .github/workflows/ci-dev.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index 1c08b90f6..ed313de1e 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -22,7 +22,9 @@ jobs: - name: Install Linux dependencies if: runner.os == 'Linux' - run: sudo apt-get install libopenblas-dev + run: | + sudo apt-get install libopenblas-dev gcc-9 g++-9 + - name: Install macOS Dependencies shell: bash -l {0} @@ -40,10 +42,18 @@ jobs: auto-activate-base: true miniforge-version: latest - - name: Install package + - name: Install package (Linux) shell: bash -l {0} - run: | - ./install_shapepipe --env-dev --develop + env: + CC: gcc-9 + CXX: g++-9 + if: runner.os == 'Linux' + run: ./install_shapepipe --env-dev --develop + + - name: Install package (macOS) + shell: bash -l {0} + if: runner.os == 'macOS' + run: ./install_shapepipe --env-dev --develop - name: Run tests shell: bash -l {0} From 1f243382c4cbb572481a5babd1cc2a7891727d7e Mon Sep 17 00:00:00 2001 From: sfarrens Date: Tue, 22 Oct 2024 17:51:59 +0200 Subject: [PATCH 20/25] add Dockerfile and corresponding workflow --- .github/workflows/ci-dev.yml | 5 +- .github/workflows/ci-release.yml | 41 ++-- .github/workflows/deploy-image.yml | 41 ++++ Dockerfile | 49 ++++ environment.yml | 337 +++++++++++++++++++++++--- install_shapepipe | 10 +- scripts/sh/canfar_submit_selection.sh | 0 setup.cfg.old | 16 -- setup.py.old | 50 ---- 9 files changed, 433 insertions(+), 116 deletions(-) create mode 100644 .github/workflows/deploy-image.yml create mode 100644 Dockerfile mode change 100644 => 100755 scripts/sh/canfar_submit_selection.sh delete mode 100644 setup.cfg.old delete mode 100644 setup.py.old diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml index ed313de1e..a9c63ee16 100644 --- a/.github/workflows/ci-dev.yml +++ b/.github/workflows/ci-dev.yml @@ -25,7 +25,6 @@ jobs: run: | sudo apt-get install libopenblas-dev gcc-9 g++-9 - - name: Install macOS Dependencies shell: bash -l {0} if: runner.os == 'macOS' @@ -48,12 +47,12 @@ jobs: CC: gcc-9 CXX: g++-9 if: runner.os == 'Linux' - run: ./install_shapepipe --env-dev --develop + run: ./install_shapepipe --env-dev - name: Install package (macOS) shell: bash -l {0} if: runner.os == 'macOS' - run: ./install_shapepipe --env-dev --develop + run: ./install_shapepipe --env-dev - name: Run tests shell: bash -l {0} diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index ef1f6a013..9ad134bbd 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -3,8 +3,7 @@ name: CI on: pull_request: branches: - - main - - master + - develop jobs: @@ -12,40 +11,52 @@ jobs: name: Full Test Suite runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash -l {0} - strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python-version: [3.9] + python-version: [3.11] steps: - - name: Checkout - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install Linux dependencies if: runner.os == 'Linux' - run: sudo apt-get install libopenblas-dev + run: | + sudo apt-get install libopenblas-dev gcc-9 g++-9 - name: Install macOS Dependencies + shell: bash -l {0} if: runner.os == 'macOS' - run: brew install libomp + run: | + brew tap sfarrens/sf + brew install bigmac libomp + brew reinstall gcc - name: Set up conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.python-version }} auto-activate-base: true + miniforge-version: latest + + - name: Install package (Linux) + shell: bash -l {0} + env: + CC: gcc-9 + CXX: g++-9 + if: runner.os == 'Linux' + run: ./install_shapepipe --develop - - name: Install package + - name: Install package (macOS) + shell: bash -l {0} + if: runner.os == 'macOS' run: ./install_shapepipe --develop - name: Run tests + shell: bash -l {0} run: | - conda activate shapepipe - python setup.py test + conda activate shapepipe-dev + pytest shapepipe_run -c example/config.ini diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml new file mode 100644 index 000000000..81c31c7ad --- /dev/null +++ b/.github/workflows/deploy-image.yml @@ -0,0 +1,41 @@ +name: Create and publish a Docker image + +on: + push: + branches: ['master'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..89e87b1a7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,49 @@ +FROM condaforge/miniforge3 + +LABEL Description="ShapePipe Docker Image" +ENV SHELL /bin/bash + +ARG CC=gcc-9 +ARG CXX=g++-9 + +# gcc < 10 is required to compile ww +ENV CC=gcc-9 +ENV CXX=g++-9 + +RUN apt-get update --allow-releaseinfo-change && \ + apt-get update && \ + apt-get upgrade -y && \ + apt-get install apt-utils -y && \ + apt-get install make -y && \ + apt-get install automake -y && \ + apt-get install autoconf -y && \ + apt-get install gcc-9 g++-9 -y && \ + # apt-get install locales -y && \ + apt-get install libgl1-mesa-glx -y && \ + # apt-get install xterm -y && \ + # apt-get install cmake protobuf-compiler -y && \ + apt-get install libtool libtool-bin libtool-doc -y && \ + apt-get install libfftw3-bin libfftw3-dev -y && \ + apt-get install libatlas-base-dev liblapack-dev libblas-dev -y && \ + apt-get install vim -y && \ + # apt-get install locate -y && \ + apt-get install curl -y && \ + # apt-get install acl -y && \ + # apt-get install sssd -y && \ + apt-get clean + +# ADD nsswitch.conf /etc/ + +# RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \ +# locale-gen +# ENV LANG en_US.UTF-8 +# ENV LANGUAGE en_US:en +# ENV LC_ALL en_US.UTF-8 + +SHELL ["/bin/bash", "--login", "-c"] + +COPY . ./shapepipe + +# RUN conda update -n base -c defaults conda -c defaults + +# RUN ./install_shapepipe --env-dev diff --git a/environment.yml b/environment.yml index ec8fae7a1..103e485e4 100644 --- a/environment.yml +++ b/environment.yml @@ -7,33 +7,314 @@ name: shapepipe channels: - conda-forge + - defaults dependencies: - - python=3.9 - - pip>=21.2.4 - - numpy==1.21.6 - - astropy==5.0 - - automake==1.16.2 - - autoconf==2.69 - - cmake==3.19.6 - - galsim==2.2.5 - - joblib==1.1.0 - - libtool==2.4.6 - - matplotlib==3.5.1 - - numba==0.54.1 - - pandas==1.4.1 + - asciitree=0.3.3=py_2 + - astromatic-source-extractor=2.28.0=h0891afa_5 + - astropy-healpix=1.0.3=py311h0f07fe1_2 + - astropy-iers-data=0.2024.10.14.0.32.55=pyhd8ed1ab_0 + - autoconf=2.71=pl5321hcd07c0c_1 + - automake=1.17=pl5321hce30654_0 + - aws-c-auth=0.7.31=h14f56dd_2 + - aws-c-cal=0.7.4=hd45b2be_2 + - aws-c-common=0.9.29=h7ab814d_0 + - aws-c-compression=0.2.19=hd45b2be_2 + - aws-c-event-stream=0.4.3=hdf5079d_4 + - aws-c-http=0.8.10=h4588aaf_2 + - aws-c-io=0.14.19=h5ad5fc2_1 + - aws-c-mqtt=0.10.7=hbe077eb_2 + - aws-c-s3=0.6.7=h86d2b7d_0 + - aws-c-sdkutils=0.1.19=hd45b2be_4 + - aws-checksums=0.1.20=hd45b2be_1 + - aws-crt-cpp=0.28.3=h4f9f7e0_8 + - aws-sdk-cpp=1.11.407=h880863c_1 + - azure-core-cpp=1.14.0=hd50102c_0 + - azure-identity-cpp=1.10.0=hc602bab_0 + - azure-storage-blobs-cpp=12.13.0=h7585a09_1 + - azure-storage-common-cpp=12.8.0=h9ca1f76_1 + - azure-storage-files-datalake-cpp=12.12.0=hcdd55da_1 + - bokeh=3.6.0=pyhd8ed1ab_0 + - brotli=1.1.0=hd74edd7_2 + - brotli-bin=1.1.0=hd74edd7_2 + - brotli-python=1.1.0=py311h3f08180_2 + - bzip2=1.0.8=h99b78c6_7 + - c-ares=1.34.2=h7ab814d_0 + - ca-certificates=2024.8.30=hf0a4a13_0 + - certifi=2024.8.30=pyhd8ed1ab_0 + - cffi=1.17.1=py311h3a79f62_0 + - cfitsio=4.4.1=hd313823_1 + - click=8.1.7=unix_pyh707e725_0 + - cloudpickle=3.1.0=pyhd8ed1ab_1 + - cmake=3.30.5=hfbcbe4a_0 + - colorama=0.4.6=pyhd8ed1ab_0 + - contourpy=1.3.0=py311h2c37856_2 + - cycler=0.12.1=pyhd8ed1ab_0 + - cytoolz=1.0.0=py311h460d6c5_1 + - dask=2024.10.0=pyhd8ed1ab_0 + - dask-core=2024.10.0=pyhd8ed1ab_0 + - dask-expr=1.1.16=pyhd8ed1ab_0 + - distributed=2024.10.0=pyhd8ed1ab_0 + - eigen=3.4.0=h1995070_0 + - fasteners=0.17.3=pyhd8ed1ab_0 + - fftw=3.3.10=nompi_h6637ab6_110 + - fitsio=1.2.4=py311h3b7b33b_1 + - fonttools=4.54.1=py311h460d6c5_0 + - freetype=2.12.1=hadb7bae_2 + - fsspec=2024.9.0=pyhff2d567_0 + - future=1.0.0=pyhd8ed1ab_0 + - galsim=2.6.0=py311h11a626f_2 + - geos=3.13.0=hf9b8971_0 + - gettext=0.22.5=h8414b35_3 + - gettext-tools=0.22.5=h8414b35_3 + - gflags=2.2.2=hf9b8971_1005 + - glog=0.7.1=heb240a5_0 + - h2=4.1.0=pyhd8ed1ab_0 + - hpack=4.0.0=pyh9f0ad1d_0 + - hyperframe=6.0.1=pyhd8ed1ab_0 + - icu=75.1=hfee45f7_0 + - importlib-metadata=8.5.0=pyha770c72_0 + - importlib_metadata=8.5.0=hd8ed1ab_0 + - jinja2=3.1.4=pyhd8ed1ab_0 + - joblib=1.4.2=pyhd8ed1ab_0 + - kiwisolver=1.4.7=py311h2c37856_0 + - krb5=1.21.3=h237132a_0 + - lcms2=2.16=ha0e7c42_0 + - lerc=4.0.0=h9a09cb3_0 + - libabseil=20240722.0=cxx17_hf9b8971_1 + - libarrow=17.0.0=h2432dd3_22_cpu + - libarrow-acero=17.0.0=h286801f_22_cpu + - libarrow-dataset=17.0.0=h286801f_22_cpu + - libarrow-substrait=17.0.0=hdcc9e87_22_cpu + - libasprintf=0.22.5=h8414b35_3 + - libasprintf-devel=0.22.5=h8414b35_3 + - libblas=3.9.0=24_osxarm64_accelerate + - libbrotlicommon=1.1.0=hd74edd7_2 + - libbrotlidec=1.1.0=hd74edd7_2 + - libbrotlienc=1.1.0=hd74edd7_2 + - libcblas=3.9.0=24_osxarm64_accelerate + - libcrc32c=1.1.2=hbdafb3b_0 + - libcurl=8.10.1=h13a7ad3_0 + - libcxx=19.1.2=ha82da77_0 + - libdeflate=1.22=hd74edd7_0 + - libedit=3.1.20191231=hc8eb9b7_2 + - libev=4.33=h93a5062_2 + - libevent=2.1.12=h2757513_1 + - libexpat=2.6.3=hf9b8971_0 + - libffi=3.4.2=h3422bc3_5 + - libgettextpo=0.22.5=h8414b35_3 + - libgettextpo-devel=0.22.5=h8414b35_3 + - libgfortran=5.0.0=13_2_0_hd922786_3 + - libgfortran5=13.2.0=hf226fd6_3 + - libgoogle-cloud=2.30.0=h2e6cea1_0 + - libgoogle-cloud-storage=2.30.0=h90fd6fa_0 + - libgrpc=1.65.5=h3d9cf25_0 + - libhwloc=2.11.2=default_h3f80f97_1000 + - libiconv=1.17=h0d3ecfb_2 + - libidn2=2.3.7=h93a5062_0 + - libintl=0.22.5=h8414b35_3 + - libintl-devel=0.22.5=h8414b35_3 + - libjpeg-turbo=3.0.0=hb547adb_1 + - liblapack=3.9.0=24_osxarm64_accelerate + - libllvm14=14.0.6=hd1a9a77_4 + - libnghttp2=1.58.0=ha4dd798_1 + - libopenblas=0.3.28=openmp_h517c56d_0 + - libparquet=17.0.0=hda0ea68_22_cpu + - libpng=1.6.44=hc14010f_0 + - libprotobuf=5.27.5=h53f8970_2 + - libre2-11=2024.07.02=h2348fd5_1 + - libsqlite=3.46.1=hc14010f_0 + - libssh2=1.11.0=h7a5bd25_0 + - libthrift=0.21.0=h64651cc_0 + - libtiff=4.7.0=hfce79cd_1 + - libtool=2.4.7=h00cdb27_1 + - libunistring=0.9.10=h3422bc3_0 + - libutf8proc=2.8.0=h1a8c8d9_0 + - libuv=1.49.2=h7ab814d_0 + - libwebp-base=1.4.0=h93a5062_0 + - libxcb=1.17.0=hdb1d25a_0 + - libxml2=2.12.7=h01dff8b_4 + - libzlib=1.3.1=h8359307_2 + - llvm-openmp=19.1.2=hb52a8e5_0 + - llvmlite=0.43.0=py311hc367efa_1 + - locket=1.0.0=pyhd8ed1ab_0 + - lsstdesc.coord=1.3.0=py311h2c37856_2 + - lz4=4.3.3=py311hebe0b09_1 + - lz4-c=1.9.4=hb7217d7_0 + - m4=1.4.18=h642e427_1001 + - markupsafe=3.0.1=py311h0ecf0c1_1 + - matplotlib=3.9.2=py311ha1ab1f8_1 + - matplotlib-base=3.9.2=py311hbe3227e_1 + - mpi=1.0=openmpi + - mpi4py=4.0.1=py311h8dda74a_0 + - msgpack-python=1.1.0=py311h2c37856_0 + - munkres=1.1.4=pyh9f0ad1d_0 + - ncurses=6.5=h7bae524_1 + - numba=0.60.0=py311h9506ed5_0 + - numcodecs=0.13.1=py311h3228b58_0 + - numpy=1.26.4=py311h7125741_0 + - openblas=0.3.28=openmp_h1f0b1b2_0 + - openjpeg=2.5.2=h9f1df11_0 + - openmpi=5.0.5=h5750160_103 + - openssl=3.3.2=h8359307_0 + - orc=2.0.2=h4a9587e_1 + - packaging=24.1=pyhd8ed1ab_0 + - pandas=2.2.3=py311h9cb3ce9_1 + - partd=1.4.2=pyhd8ed1ab_0 + - perl=5.32.1=7_h4614cfb_perl5 + - pillow=11.0.0=py311h3894ae9_0 + - pip=24.2=pyh8b19718_1 + - ply=3.11=pyhd8ed1ab_2 + - psutil=6.0.0=py311hae2e1ce_2 + - pthread-stubs=0.4=hd74edd7_1002 + - pyarrow=17.0.0=py311h35c05fe_1 + - pyarrow-core=17.0.0=py311he42f270_1_cpu + - pybind11=2.13.6=pyh085cc03_1 + - pybind11-global=2.13.6=pyh085cc03_1 + - pycparser=2.22=pyhd8ed1ab_0 + - pyerfa=2.0.1.4=py311h0f07fe1_2 + - pyparsing=3.2.0=pyhd8ed1ab_1 + - pyqt5-sip=12.12.2=py311ha891d26_5 + - pyqtgraph=0.13.7=pyhd8ed1ab_0 + - pysocks=1.7.1=pyha2e5f31_6 + - python=3.11.10=hc51fdd5_3_cpython + - python-dateutil=2.9.0=pyhd8ed1ab_0 + - python-tzdata=2024.2=pyhd8ed1ab_0 + - python_abi=3.11=5_cp311 + - pytz=2024.1=pyhd8ed1ab_0 + - pyyaml=6.0.2=py311h460d6c5_1 + - qhull=2020.2=h420ef59_5 + - re2=2024.07.02=hcd0e937_1 + - readline=8.2=h92ec313_1 + - reproject=0.14.0=py311h5d790af_0 + - rhash=1.4.5=h7ab814d_0 + - scipy=1.14.1=py311h2929bc6_0 + - setuptools=75.1.0=pyhd8ed1ab_0 + - shapely=2.0.6=py311hac502b4_2 + - sip=6.8.6=py311h3f08180_1 + - six=1.16.0=pyh6c4a22f_0 + - snappy=1.2.1=hd02b534_0 + - sortedcontainers=2.4.0=pyhd8ed1ab_0 + - sqlitedict=2.1.0=pyhd8ed1ab_0 + - tblib=3.0.0=pyhd8ed1ab_0 + - termcolor=2.5.0=pyhd8ed1ab_0 + - tk=8.6.13=h5083fa2_1 + - toml=0.10.2=pyhd8ed1ab_0 + - tomli=2.0.2=pyhd8ed1ab_0 + - toolz=1.0.0=pyhd8ed1ab_0 + - tornado=6.4.1=py311h460d6c5_1 + - tqdm=4.66.5=pyhd8ed1ab_0 + - treecorr=5.0.1=py311h356bf30_1 + - tzdata=2024b=hc8b5060_0 + - urllib3=2.2.3=pyhd8ed1ab_0 + - wget=1.21.4=he2df1f1_0 + - wheel=0.44.0=pyhd8ed1ab_0 + - xorg-libxau=1.0.11=hd74edd7_1 + - xorg-libxdmcp=1.1.5=hd74edd7_0 + - xyzservices=2024.9.0=pyhd8ed1ab_0 + - xz=5.2.6=h57fd34a_0 + - yaml=0.2.5=h3422bc3_2 + - zarr=2.18.3=pyhd8ed1ab_0 + - zict=3.0.0=pyhd8ed1ab_0 + - zipp=3.20.2=pyhd8ed1ab_0 + - zlib=1.3.1=h8359307_2 + - zstandard=0.23.0=py311ha60cc69_1 + - zstd=1.5.6=hb46c0d2_0 - pip: - - cs_util==0.0.5 - - mccd==1.2.3 - - modopt==1.6.0 - - PyQt5==5.15.6 - - pyqtgraph==0.12.4 - - python-pysap==0.0.6 - - reproject==0.8 - - sip_tpv==1.1 - - sf_tools==2.0.4 - - sqlitedict==2.0.0 - - termcolor==1.1.0 - - tqdm==4.63.0 - - treecorr==4.2.6 - - git+https://github.com/aguinot/ngmix@stable_version - - git+https://github.com/tobias-liaudat/Stile@v0.1 + - accessible-pygments==0.0.5 + - aenum==3.1.15 + - alabaster==1.0.0 + - astropy==5.2 + - babel==2.16.0 + - backports-tarfile==1.2.0 + - beautifulsoup4==4.12.3 + - black==24.10.0 + - build==1.2.2.post1 + - cadcutils==1.5.3 + - camb==1.5.4 + - charset-normalizer==3.4.0 + - coverage==7.6.3 + - cryptography==43.0.3 + - cs-util==0.1.0 + - datetime==5.5 + - distro==1.9.0 + - docutils==0.21.2 + - html2text==2024.2.26 + - idna==3.10 + - imageio==2.36.0 + - imagesize==1.4.1 + - importlib-resources==6.4.5 + - iniconfig==2.0.0 + - isort==5.13.2 + - jaraco-classes==3.4.0 + - jaraco-context==6.0.1 + - jaraco-functools==4.1.0 + - keyring==25.4.1 + - latexcodec==3.0.0 + - lazy-loader==0.4 + - lxml==5.3.0 + - markdown-it-py==3.0.0 + - mccd==1.2.4 + - mdit-py-plugins==0.4.2 + - mdurl==0.1.2 + - modopt==1.7.2 + - more-itertools==10.5.0 + - mpmath==1.3.0 + - mypy-extensions==1.0.0 + - myst-parser==4.0.0 + - networkx==3.4.1 + - ngmix==1.3.6 + - nh3==0.2.18 + - nibabel==5.3.1 + - numpydoc==1.8.0 + - pathspec==0.12.1 + - pkginfo==1.10.0 + - platformdirs==4.3.6 + - pluggy==1.5.0 + - progressbar2==4.5.0 + - py==1.11.0 + - pybtex==0.24.0 + - pybtex-docutils==1.0.3 + - pycodestyle==2.12.1 + - pydata-sphinx-theme==0.15.4 + - pydocstyle==6.3.0 + - pygments==2.18.0 + - pyopenssl==24.2.1 + - pyproject-hooks==1.2.0 + - pytest==8.3.3 + - pytest-cov==5.0.0 + - pytest-pycodestyle==2.4.1 + - pytest-pydocstyle==2.4.0 + - python-pysap==0.2.1 + - python-utils==3.9.0 + - pywavelets==1.7.0 + - readme-renderer==44.0 + - requests==2.32.3 + - requests-toolbelt==1.0.0 + - rfc3986==2.0.0 + - rich==13.9.2 + - scikit-image==0.24.0 + - scikit-learn==1.5.2 + - sf-tools==2.0.4 + - shapepipe==1.0.2 + - sip-tpv==1.1 + - snowballstemmer==2.2.0 + - soupsieve==2.6 + - sphinx==8.1.3 + - sphinx-book-theme==1.1.3 + - sphinxcontrib-applehelp==2.0.0 + - sphinxcontrib-bibtex==2.6.3 + - sphinxcontrib-devhelp==2.0.0 + - sphinxcontrib-htmlhelp==2.1.0 + - sphinxcontrib-jsmath==1.0.1 + - sphinxcontrib-qthelp==2.0.0 + - sphinxcontrib-serializinghtml==2.0.0 + - stile==0.1 + - swig==4.2.1 + - sympy==1.13.3 + - tabulate==0.9.0 + - threadpoolctl==3.5.0 + - tifffile==2024.9.20 + - twine==5.1.1 + - typing-extensions==4.12.2 + - vos==3.6.1.1 + - zope-interface==7.1.0 diff --git a/install_shapepipe b/install_shapepipe index 88ec7a2a6..0f6d6e4f0 100755 --- a/install_shapepipe +++ b/install_shapepipe @@ -10,7 +10,7 @@ export LC_CTYPE=en_US.utf8 # ############################################################################## -version="1.7" +version="1.8" last_update="13/10/24" ############################################################################## @@ -181,7 +181,6 @@ check_conda() { CONDA_ACTIVATE="conda activate" CONDA_DEACTIVATE="conda deactivate" fi - } # Check if a binary executable is already installed in the conda environment @@ -298,8 +297,11 @@ exit_code() { # Function to report package versions package_report() { + + check_env_name if [ "$CONDA_DEFAULT_ENV" != "$ENV_NAME" ] then + check_conda activate_shapepipe fi INSTALL_PSFEX=FALSE @@ -451,9 +453,9 @@ Options:\n \t--pipe-dir\t set the path to the ShapePipe base directory (default is \$PWD)\n \t--build-dir\t set the path to the ShapePipe build (default is \$PWD/build)\n \t--env-name\t set the name of the Conda environment (default is shapepipe)\n -\t--env-dev\t use the developers Conda environment instead of the standard production environment\n +\t--env-dev\t use the developer's Conda environment instead of the standard production environment\n \t--use-conda-source\t use 'conda source' instead of the default 'conda activate' to activate the shapepipe environment\n -\t--develop\t option to install additional development packages\n +\t--develop\t option to install additional development packages (included by default in --env-dev)\n \t--vos\t\t option to install virtual observatory software\n \t--no-env\t do not build Conda environment\n \t--no-psfex\t do not build PSFEx\n diff --git a/scripts/sh/canfar_submit_selection.sh b/scripts/sh/canfar_submit_selection.sh old mode 100644 new mode 100755 diff --git a/setup.cfg.old b/setup.cfg.old deleted file mode 100644 index 9979e89ff..000000000 --- a/setup.cfg.old +++ /dev/null @@ -1,16 +0,0 @@ -[aliases] -test=pytest - -[metadata] -description-file = README.rst - -[tool:pytest] -testpaths = shapepipe -addopts = --verbose --pycodestyle --pydocstyle --cov=shapepipe - -[pycodestyle] -ignore = W503 - -[pydocstyle] -convention = numpy -add-ignore = D105,D107 diff --git a/setup.py.old b/setup.py.old deleted file mode 100644 index e5610581e..000000000 --- a/setup.py.old +++ /dev/null @@ -1,50 +0,0 @@ -#! /usr/bin/env python - -from setuptools import setup, find_packages -import os - -package_info = {} -infopath = os.path.abspath( - os.path.join(os.path.dirname(__file__), 'shapepipe', 'info.py') -) -with open(infopath) as open_file: - exec(open_file.read(), package_info) - - -# Find scripts -def find_scripts(): - - scripts = [] - scripts_dir = package_info['__scripts_dir__'] - valid_extensions = package_info['__scripts_ext__'] - - sub_dirs = ( - os.path.join(scripts_dir, sub_dir) - for sub_dir in os.listdir(scripts_dir) - if os.path.isdir(os.path.join(scripts_dir, sub_dir)) - ) - - for sub_dir in sub_dirs: - scripts.extend([ - os.path.join(sub_dir, val) for val in os.listdir(sub_dir) - if os.path.isfile(os.path.join(sub_dir, val)) - and '__init__' not in val - and val.endswith(valid_extensions) - ]) - - return scripts - - -setup( - name=package_info['__name__'], - author=package_info['__author__'], - author_email=package_info['__email__'], - version=package_info['__version__'], - packages=find_packages(), - scripts=find_scripts(), - setup_requires=package_info['__setups__'], - install_requires=package_info['__installs__'], - description='Galaxy shape measurement pipeline.', - long_description=package_info['__about__'], - tests_require=package_info['__tests__'], -) From dea1f547200d607865d3e13a6cefd070a90a7389 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Wed, 23 Oct 2024 10:52:58 +0200 Subject: [PATCH 21/25] update CDS version --- install_shapepipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_shapepipe b/install_shapepipe index 0f6d6e4f0..4bc01a6f0 100755 --- a/install_shapepipe +++ b/install_shapepipe @@ -45,7 +45,7 @@ ww_version=TRUE # CDSclient Package (need Findgsc2.2) cdsclient_url="http://cdsarc.u-strasbg.fr/ftp/pub/sw/cdsclient.tar.gz" cdscleint_tar="cdsclient.tar.gz" -cdsclient_dir="cdsclient-3.84" +cdsclient_dir="cdsclient-4.07" cdsclient_bin="findgsc2.2" cdsclient_version=FALSE From 25cd93a28a54935264e9f8a302451f0f2c742fa8 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Wed, 23 Oct 2024 10:53:43 +0200 Subject: [PATCH 22/25] fix release CI trigger --- .github/workflows/ci-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index 9ad134bbd..82445dfd0 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -3,7 +3,8 @@ name: CI on: pull_request: branches: - - develop + - main + - master jobs: From f505df264a5c8fa926d3bdf2996bae6b897ede67 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Wed, 23 Oct 2024 16:27:29 +0200 Subject: [PATCH 23/25] removed Dockerfile and corresponding workflow --- .github/workflows/deploy-image.yml | 41 ------------------------- Dockerfile | 49 ------------------------------ 2 files changed, 90 deletions(-) delete mode 100644 .github/workflows/deploy-image.yml delete mode 100644 Dockerfile diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml deleted file mode 100644 index 81c31c7ad..000000000 --- a/.github/workflows/deploy-image.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Create and publish a Docker image - -on: - push: - branches: ['master'] - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build-and-push-image: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Log in to the Container registry - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 89e87b1a7..000000000 --- a/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -FROM condaforge/miniforge3 - -LABEL Description="ShapePipe Docker Image" -ENV SHELL /bin/bash - -ARG CC=gcc-9 -ARG CXX=g++-9 - -# gcc < 10 is required to compile ww -ENV CC=gcc-9 -ENV CXX=g++-9 - -RUN apt-get update --allow-releaseinfo-change && \ - apt-get update && \ - apt-get upgrade -y && \ - apt-get install apt-utils -y && \ - apt-get install make -y && \ - apt-get install automake -y && \ - apt-get install autoconf -y && \ - apt-get install gcc-9 g++-9 -y && \ - # apt-get install locales -y && \ - apt-get install libgl1-mesa-glx -y && \ - # apt-get install xterm -y && \ - # apt-get install cmake protobuf-compiler -y && \ - apt-get install libtool libtool-bin libtool-doc -y && \ - apt-get install libfftw3-bin libfftw3-dev -y && \ - apt-get install libatlas-base-dev liblapack-dev libblas-dev -y && \ - apt-get install vim -y && \ - # apt-get install locate -y && \ - apt-get install curl -y && \ - # apt-get install acl -y && \ - # apt-get install sssd -y && \ - apt-get clean - -# ADD nsswitch.conf /etc/ - -# RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \ -# locale-gen -# ENV LANG en_US.UTF-8 -# ENV LANGUAGE en_US:en -# ENV LC_ALL en_US.UTF-8 - -SHELL ["/bin/bash", "--login", "-c"] - -COPY . ./shapepipe - -# RUN conda update -n base -c defaults conda -c defaults - -# RUN ./install_shapepipe --env-dev From 6d40db76a4df97977d531285b6ed399230cf0456 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Tue, 29 Oct 2024 14:07:59 +0100 Subject: [PATCH 24/25] fix licence --- LICENSE | 96 ++------------------------------------------------------- 1 file changed, 2 insertions(+), 94 deletions(-) diff --git a/LICENSE b/LICENSE index 2b83262ae..5e8d01f21 100644 --- a/LICENSE +++ b/LICENSE @@ -1,18 +1,6 @@ -# LICENCES +MIT License -Le code source de ce dépôt est publié sous [licence MIT](#licence-mit). - -Sauf mention de propriété intellectuelle détenue par des tiers (notamment -un crédit sur certaines images), les contenus de ce dépôt sont publiés sous [licence Ouverte 2.0](#licence-ouverte-20open-licence-20). - -La marque d'État est à usage exclusif des acteurs de la sphère -étatique. En particulier, la typographie Marianne© est protégée par -le droit d'auteur. [Lire les explications sur le site de la marque -d'État.](https://www.gouvernement.fr/charte/charte-graphique-les-fondamentaux/la-typographie) - -## LICENCE MIT - -Copyright (c) 2021 DINUM +Copyright (c) 2024 CosmoStat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -31,83 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -## LICENCE OUVERTE 2.0/OPEN LICENCE 2.0 - -### Réutilisation de l’« Information » sous cette licence - -Le « Concédant » concède au « Réutilisateur » un droit non exclusif et gratuit de libre « Réutilisation » de l’« Information » objet de la présente licence, à des fins commerciales ou non, dans le monde entier et pour une durée illimitée, dans les conditions exprimées ci-dessous. - -**Le « Réutilisateur » est libre de réutiliser l’« Information » :** - -- de la communiquer, la reproduire, la copier ; -- de l’adapter, la modifier, l’extraire et la transformer, notamment pour créer des « Informations dérivées » ; -- de la diffuser, la redistribuer, la publier et la transmettre, de l’exploiter à titre commercial, par exemple en la combinant avec d’autres informations, ou en l’incluant dans votre propre produit ou application. - -**Sous réserve de :** - -- mentionner la paternité de l’«Information» : sa source (a minima le nom du « Concédant ») et la date de la dernière mise à jour de l’« Information » réutilisée. - -Le « Réutilisateur » peut notamment s’ acquitter de cette condition en indiquant l’adresse (URL) renvoyant vers « l’Information » et assurant une mention effective de sa paternité. - -**Par exemple :** - -Dans le cas d’une réutilisation de la base SIRENE de l’INSEE, mentionner l’URL du « Concédant » : www.insee.fr + la date de dernière mise à jour de l’Information réutilisée. - -Cette mention de paternité ne doit ni conférer un caractère officiel à la « Réutilisation » de l’« Information », ni suggérer une quelconque reconnaissance ou caution par le « Concédant », ou par toute autre entité publique, du « Réutilisateur » ou de sa « Réutilisation ». - -### Données à caractère personnel - -L’« Information » mise à disposition peut contenir des « Données à caractère personnel » pouvant faire l’objet d’une « Réutilisation ». Alors, le « Concédant » informe le « Réutilisateur » (par tous moyens) de leur présence, l’ « Information » peut être librement réutilisée, sans faire obstacle aux libertés accordées par la présente licence, à condition de respecter le cadre légal relatif à la protection des données à caractère personnel. - -### Droits de propriété intellectuelle - -Il est garanti au « Réutilisateur » que l’ « Information » ne contient pas de « Droits de propriété intellectuelle » appartenant à des tiers qui pourraient faire obstacle aux libertés qui lui sont accordées par la présente licence. - -Les éventuels « Droits de propriété intellectuelle » détenus par le « Concédant » sur l’ « Information » ne font pas obstacle aux libertés qui sont accordées par la présente licence. Lorsque le « Concédant » détient des « Droits de propriété intellectuelle » » sur l’ « Information », il les cède au « Réutilisateur » de façon non exclusive, à titre gracieux, pour le monde entier, pour toute la durée des « Droits de propriété intellectuelle », et le « Réutilisateur » peut en faire tout usage conformément aux libertés et aux conditions définies par la présente licence. - -### Responsabilité - -L’ «Information» est mise à disposition telle que produite ou reçue, sans autre garantie expresse ou tacite qui n’est pas prévue par la présente licence. L’absence de défauts ou d’erreurs éventuellement contenues dans l’ «Information», comme la fourniture continue de l’ « Information » n’est pas garantie par le «Concédant». Il ne peut être tenu pour responsable de toute perte, préjudice ou dommage de quelque sorte causé à des tiers du fait de la « Réutilisation ». - -Le « Réutilisateur » est seul responsable de la « Réutilisation » de l’« Information ». - -La « Réutilisation » ne doit pas induire en erreur des tiers quant au contenu de l’« Information », sa source et sa date de mise à jour. - -### Droit applicable - -La présente licence est régie par le droit français. - -#### Compatibilité de la présente licence - -Elle a été conçue pour être compatible avec toute licence libre qui exige _a minima_ la mention de paternité. Elle est notamment compatible avec la version antérieure de la présente licence ainsi qu’avec les licences « Open Government Licence » (OGL) du Royaume-Uni, « Creative Commons Attribution » (CC-BY) de Creative Commons et « Open Data Commons Attribution » (ODC-BY) de l’Open Knowledge Foundation. - -### Définitions - -Sont considérés, au sens de la présente licence comme : - -- Le « **Concédant** » : toute personne concédant un droit de « Réutilisation » sur l’« Information » dans les libertés et les conditions prévues par la présente licence. -- L’« **Information** » : - - toute information publique figurant dans des documents communiqués ou publiés par une administration mentionnée au premier alinéa de l’article L.300-2 du CRPA ; - - toute information mise à disposition par toute personne selon les termes et conditions de la présente licence. -- La « **Réutilisation** » : l’utilisation de l’« Information » à d’autres fins que celles pour lesquelles elle a été produite ou reçue. -- Le « **Réutilisateur** » : toute personne qui réutilise les « Informations » conformément aux conditions de la présente licence. -- Des « **Données à caractère personnel** » : toute information se rapportant à une personne physique identifiée ou identifiable, pouvant être identifiée directement ou indirectement. Leur « Réutilisation » est subordonnée au respect du cadre juridique en vigueur. -- Une « **Information dérivée** » : toute nouvelle donnée ou information créées directement à partir de l’« Information » ou à partir d’une combinaison de l’ « Information » et d’autres données ou informations non soumises à cette licence. -- Les « **Droits de propriété intellectuelle** » : tous droits identifiés comme tels par le Code de la propriété intellectuelle (droit d’auteur, droits voisins au droit d’auteur, droit sui generis des producteurs de bases de données). - -### À propos de cette licence - -La présente licence a vocation à être utilisée par les administrations pour la réutilisation de leurs informations publiques. Elle peut également être utilisée par toute personne souhaitant mettre à disposition de l’« Information » dans les conditions définies par la présente licence - -La France est dotée d’un cadre juridique global visant à une diffusion spontanée par les administrations de leurs informations publiques afin d’en permettre la plus large réutilisation. - -Le droit de la « Réutilisation » de l’« Information » des administrations est régi par le code des relations entre le public et l’administration (CRPA) et, le cas échéant, le code du patrimoine (livre II relatif aux archives). - -Cette licence facilite la réutilisation libre et gratuite des informations publiques et figure parmi les licences qui peuvent être utilisées par l’administration en vertu du décret pris en application de l’article L.323-2 du CRPA. - -Etalab est la mission chargée, sous l’autorité du Premier ministre, d’ouvrir le plus grand nombre de données publiques des administrations de l’État et de ses établissements publics. Elle a réalisé la Licence Ouverte pour faciliter la réutilisation libre et gratuite de ces informations publiques, telles que définies par l’article L321-1 du CRPA. - -Cette licence est une version 2.0 de la Licence Ouverte. - -Etalab se réserve la faculté de proposer de nouvelles versions de la Licence Ouverte. Cependant, les « Réutilisateurs » pourront continuer à réutiliser les informations disponibles sous cette licence s’ils le souhaitent. From 2f66b466ab628f3eee1106bd8f0658f23e0c1417 Mon Sep 17 00:00:00 2001 From: sfarrens Date: Tue, 29 Oct 2024 14:14:07 +0100 Subject: [PATCH 25/25] use latest release of PySAP --- environment-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment-dev.yml b/environment-dev.yml index eb268f74d..f9b747c1a 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -36,6 +36,6 @@ dependencies: - modopt>=1.6 - sip_tpv>=1.1 - sf_tools>=2.0.4 - - git+https://github.com/CEA-COSMIC/pysap@develop + - python-pysap>=0.2.1 - git+https://github.com/aguinot/ngmix@stable_version - git+https://github.com/tobias-liaudat/Stile@v0.1