monitoring

.github/workflows/ci-tests.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, centos-latest]
         python-version: [3.9]
 
     steps:

.github/workflows/deploy-image.yml

@@ -0,0 +1,41 @@
+name: Create and publish a Docker image
+
+on:
+  push:
+    branches: ['master']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -0,0 +1,57 @@
+FROM continuumio/miniconda3
+
+LABEL Description="ShapePipe Docker Image"
+ENV SHELL /bin/bash
+
+ARG CC=gcc-9
+ARG CXX=g++-9
+
+# gcc < 10 is required to compile ww
+ENV CC=gcc-9
+ENV CXX=g++-9
+
+RUN apt-get update --allow-releaseinfo-change && \
+    apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install apt-utils -y && \
+    apt-get install make -y && \
+    apt-get install automake -y && \
+    apt-get install autoconf -y && \
+    apt-get install gcc-9 g++-9 -y && \
+    apt-get install locales -y && \
+    apt-get install libgl1-mesa-glx -y && \
+    apt-get install xterm -y && \
+    apt-get install cmake protobuf-compiler -y && \
+    apt-get install libtool libtool-bin libtool-doc -y && \
+    apt-get install libfftw3-bin libfftw3-dev -y && \
+    apt-get install libatlas-base-dev liblapack-dev libblas-dev -y && \
+    apt-get install vim -y && \
+    apt-get install locate -y && \
+    apt-get install curl -y && \
+    apt-get install acl -y && \
+    apt-get install sssd -y && \
+    apt-get clean
+
+ADD nsswitch.conf /etc/
+
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
+    locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+SHELL ["/bin/bash", "--login", "-c"]
+
+COPY ./environment.yml ./
+COPY install_shapepipe README.rst setup.py setup.cfg ./
+RUN touch ./README.md
+
+RUN conda update -n base -c defaults conda -c defaults
+RUN conda env create --file environment.yml
+
+COPY shapepipe ./shapepipe
+COPY scripts ./scripts
+
+# Make RUN commands use the new environment:
+SHELL ["conda", "run", "-n", "shapepipe", "/bin/bash", "-c"]
+RUN pip install jupyter

auxdir/CFIS/tiles_202106/tiles_P7.txt

@@ -1,3 +1,4 @@
+000.000
 052.332
 053.331
 053.332

docs/source/canfar.md

@@ -0,0 +1,94 @@
+# Running `shapepipe` on the canfar science portal
+
+## Introduction
+
+## Steps from testing to parallel running
+
+Before starting a batch remote session job on a large number of images (step 5.),
+it is recommended to perform some or all of the testing steps (1. - 4.).
+
+
+1. Run the basic `shapepipe` runner script to test (one or several) modules in question, specified by a given config file, on one image.
+   This step has to be run in the image run directory. The command is
+   ```bash
+   shapepipe_run -c config.ini
+   ```
+
+2. Run the job script to test the job management, on one image.
+   This step has to be run in the image run directory. The command is
+   ```bash
+   job_sp_canfar -j JOB [OPTIONS]
+   ```
+
+3. Run the pipeline script to test the processing step(s), on one image.
+   This step has to be run in the patch base directory.
+
+   1. First, run in dry mode:
+      ```bash
+      init_run_exclusive_canfar.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n
+      ```
+   2. Next, perform a real run with
+      ```bash
+      init_run_exclusive_canfar.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n
+      ```
+
+4. Run remote session script to test job submission using docker images, on one image.
+   This step has to be run in the patch base directory.
+   1. First, run in dry mode=2, to display curl command, with
+      ```bash
+      curl_canfar_local.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n 2
+      ```
+
+   2. Next, run in dry mode=1, to use curl command without processing:
+      ```bash
+      curl_canfar_local.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n 1
+      ```
+   3. Then, perform a real run, to use curl with processing:
+      ```bash
+      curl_canfar_local.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp]
+      ```
+
+5. Full run: Call remote session script and docker image with collection of images
+      ```bash
+      curl_canfar_local.sh -j JOB -f path_IDs -p [psfex|mccd] -k [tile|exp]
+      ```
+      with `path_IDs` being a text file with one image ID per line.
+
+## Monitoring
+
+
+### Status and output of submitted job 
+
+Monitoring of the currently active remote session can be performed using the session IDs `session_IDs.txt` written by the
+remote session script `curl_canfar_local.sh`. In the patch main directory, run
+```bash
+curl_canfar_monitor.sh events
+```
+to display the remotely started docker image status, and
+```bash
+curl_canfar_monitor.sh logs
+```
+to print `stdout` of the remotely run pipeline script.
+
+### Number of submitted running jobs
+
+The script
+```bash
+stats_headless_canfar.py
+```
+returns the number of actively running headless jobs.
+
+
+## Post-hoc summary
+
+In the patch main directory, run
+```bash
+summary_run PATCH
+```
+to print a summary with missing image IDs per job and module.
+
+## Deleting jobs
+
+```bash
+ for id in `cat session_IDs.txt`; do echo $id; curl -X DELETE -E /arc/home/kilbinger/.ssl/cadcproxy.pem https://ws-uv.canfar.net/skaha/v0/session/$id;  done
+ ```

docs/source/pipeline_canfar.md

@@ -0,0 +1,91 @@
+patch="P7"
+psf="psfex"
+N_SMP=16
+
+# Terminal title
+echo -ne "\033]0;$patch\007"
+
+# Run directory
+dir=~/cosmostat/v2/pre_v2/$psf/$patch
+cd $dir
+
+# Get tile number list
+ln -s ~/shapepipe/auxdir/CFIS/tiles_202106/tiles_$patch.txt tile_numbers.txt
+
+
+# Get images
+
+## Download and link separately
+
+### Download
+### Create and link to central image storage directory
+mkdir -p ~/cosmostat/v2/data_tiles/$patch
+ln -s ~/cosmostat/v2/data_tiles/$patch data_tiles
+
+### Download and move tiles 
+ln -s ~/shapepipe/example/cfis
+mkdir -p output
+export SP_RUN=`pwd`
+
+shapepipe_run -c cfis/config_Git_vos.ini
+mv -i output/run_sp_Git_*/get_images_runner/output/CFIS.???.???.*fits* data_tiles
+rm -rf output/run_sp_tiles_Git_*
+update_run_log_file.py
+# repeat the above block
+
+### Find exposures; this run can be stopped after Fe
+shapepipe_run -c cfis/config_GitFe_symlink.ini
+
+### Download and move exposures
+
+shapepipe_run -c cfis/config_Gie_vos.ini
+mv -i output/run_sp_Gie_*/get_images_runner/output/*.fits*fz data_exp
+rm -rf  output/run_sp_Gie_*
+update_run_log_file.py
+# repeat the above
+
+### Create links (and re-run Fe, not necessary)
+job_sp_canfar.bash -p $psf `cat tile_numbers.txt` -j 1 -r symlink
+
+# Uncompress weights,  split exposures into single HDUs
+job_sp_canfar.bash -p $psf -n $N_SMP -j 2
+
+# Mask tiles
+job_sp_canfar.bash -p $psf -n $N_SMP -j 4
+
+# Mask exposures
+job_sp_canfar.bash -p $psf -n $N_SMP -j 8
+
+
+# Tile detection
+curl_canfar_local.sh -j 16 -f tile_numbers.txt -k tile -p $psf -N $N_SMP
+
+
+# Exposure detection
+## Get single-HDU single-exposure IDs
+~/shapepipe/scripts/python/summary_run.py
+
+cp summary/missing_job_32_sextractor.txt all.txt
+curl_canfar_local.sh -j 32 -f all.txt -k exp -p $psf -N $N_SMP
+
+# Tile preparation
+curl_canfar_local.sh -j 64 -f tile_numbers.txt -k tile -p $psf -N $N_SMP
+
+# Tile shape measurement
+curl_canfar_local.sh -j 128 -f tile_numbers.txt -k tile -p $psf -N 8
+
+# Merge subcatalogues, and create final cat
+job_sp_canfar.bash -p $psf -n 1 -j 256
+
+# Combine all final cats in common output dir as links
+combine_runs.bash -c final -p psfex
+
+# Merge all final cats
+# (use 192GB RAM)
+merge_final_cat -i output/run_sp_combined_final/make_catalog_runner/output -p cfis/final_cat.param -v
+
+
+# Delete jobs
+SSL=~/.ssl/cadcproxy.pem
+SESSION=https://ws-uv.canfar.net/skaha/v0/session
+for ID in `cat session_IDs.txt`; do echo $ID; curl -X DELETE -E $SSL $SESSION/$ID; done