Adding CPU changes

nasa-nccs-hpda · Sep 30, 2024 · 4d0bd20 · 4d0bd20
1 parent 5db02ab
commit 4d0bd20
Show file tree

Hide file tree

Showing 3 changed files with 188 additions and 143 deletions.
diff --git a/.github/workflows/dockerhub-dev.yaml b/.github/workflows/dockerhub-dev.yaml
@@ -1,4 +1,4 @@
-name: CI to Dockerhub
+name: CI to Dockerhub - Dev
 
 on:
   push:
@@ -52,4 +52,4 @@ jobs:
           context: .
           file: ./requirements/Dockerfile.dev
           push: true
-          tags: nasanccs/vhr-toolkit:latest
+          tags: nasanccs/vhr-toolkit:dev
diff --git a/requirements/Dockerfile-old.cpu b/requirements/Dockerfile-old.cpu
@@ -0,0 +1,183 @@
+# vhr-toolkit combined container
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.9.1
+
+# Arguments to pass to the image
+ARG STEREO_URL="https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/2.7.0/StereoPipeline-2.7.0-2020-07-29-x86_64-Linux.tar.bz2"
+ARG PROJECT_PATH="/usr/local/ilab"
+
+# Environment
+ENV PYTHONPATH="$PYTHONPATH:/usr/local/ilab"
+# why is this?????? really bad practice
+ENV PYTHONPATH="/usr/local/lib/python3.8/dist-packages"
+ENV REDIS_PORT="6379"
+ENV REDIS_FILE="/etc/profile.d/redis_server.sh"
+
+# Ubuntu needs noninteractive to be forced
+ENV DEBIAN_FRONTEND=noninteractive
+ENV CPLUS_INCLUDE_PATH="/usr/include/gdal"
+ENV C_INCLUDE_PATH="/usr/include/gdal"
+ENV PROJ_LIB="/usr/share/proj"
+ENV PROJ_DATA="/usr/share/proj"
+ENV SM_FRAMEWORK="tf.keras"
+ENV PYTHONPATH="/usr/local/lib/python3.8/dist-packages"
+ENV PYTHONPATH="$PYTHONPATH:/usr/local/ilab"
+
+ENV PYTHONPATH="$PYTHONPATH:/usr/local/ilab/vhr-cloudmask:/usr/local/ilab/evhr:/usr/local/ilab/core:/usr/local/ilab/srlite/srlite:/usr/local/ilab/srlite"
+ENV PYTHONPATH="$PYTHONPATH:/opt/DgStereo/pygeotools:/opt/DgStereo/dgtools"
+ENV PYTHONPATH="$PYTHONPATH:/opt/DgStereo/pygeotools:/opt/DgStereo/dgtools"
+ENV PATH="$PATH:/opt/DgStereo/dgtools:/opt/DgStereo/pygeotools:/opt/DgStereo/dgtools/dgtools:/opt/StereoPipeline/bin:/opt/DgStereo/evhr"
+ENV REDIS_PORT="6379"
+
+# System dependencies
+RUN apt-get update && apt-get -y dist-upgrade && \
+    apt-get -y install \
+        build-essential \
+        # EVHR system dependencies
+        parallel \
+        libdatetime-perl \
+        gawk \
+        util-linux \
+        bc \
+        # SRLite system dependencies
+        python3-tk \
+        tk-dev \
+        libgdbm-dev \
+        libc6-dev \
+        libbz2-dev \
+        libffi-dev \
+        zlib1g-dev \
+        liblzma-dev \
+        libgirepository1.0-dev \
+        libcairo2-dev \
+        pkg-config \
+        python3-dev \
+        gir1.2-gtk-3.0 && \
+    apt-get -y autoremove && \
+    rm -rf /var/cache/apt /var/lib/apt/lists/*
+
+RUN PROJECT_PATH="/usr/local/ilab" && \
+    REDIS_FILE="/etc/profile.d/redis_server.sh" && \
+    echo $PROJECT_PATH && \
+    mkdir -p $PROJECT_PATH && \
+    #-------------------------------------------------------------------------------
+    # System Dependencies
+    #-------------------------------------------------------------------------------
+    apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        python3-pip python3-dev wget vim curl git procps gcc g++ bzip2 libssl-dev \
+        libsqlite3-dev libx11-dev libgeos++-dev libproj-dev
+    DEBIAN_FRONTEND=noninteractive apt-get -y autoremove && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /var/cache/apt
+
+    #-------------------------------------------------------------------------------
+    # Python Stack
+    #-------------------------------------------------------------------------------
+RUN pip install pipx && \
+     pipx install --upgrade pip && \
+     pip install awscli s3fs && \
+     pip install celery[redis] && \
+     pip install certifi && \
+     pip install cryptography && \
+     pip install Cython && \
+     pip install dask && \
+     pip install decorator && \
+     pip install dill && \
+     pip install distributed && \
+     pip install docutils && \
+     pip install fiona && \
+     pip install flower && \
+     pip install geopandas && \
+     pip install h5py && \
+     pip install imageio && \
+     pip install ipykernel && \
+     pip install ipython && \
+     pip install ipython-genutils && \
+     pip install ipywidgets && \
+     pip install matplotlib && \
+     pip install netcdf4 && \
+     pip install networkx && \
+     pip install notebook && \
+     pip install numba && \
+     pip install numexpr && \
+     pip install numpy && \
+     pip install pandas && \
+     pip install pyhdf && \
+     pip install pyproj && \
+     pip install Pysal && \
+     pip install PyYAML && \
+     pip install rasterio && \
+     pip install redis && \
+     pip install requests && \
+     pip install rioxarray && \
+     pip install scikit-image && \
+     pip install scikit-learn && \
+     pip install scipy && \
+     pip install seaborn && \
+     pip install shapely && \
+     pip install cartopy && \
+     pip install xarray && \
+     pip install urllib3 && \
+     pip install zarr && \
+     pip install earthengine-api && \
+     pip install geemap && \
+     pip install geedim && \
+     pip install vhr-cloudmask && \
+     pip cache purge
+
+#    # Add redis-server binary to /usr/local/bin
+RUN export pyVer=`python --version | awk -F' ' '{print $2}' | awk -F'.' '{print $1"."$2}'` && \
+    ln -sf /usr/local/lib/python${pyVer}/dist-packages/redis_server/bin/redis-server /usr/local/bin/redis-server && \
+    echo "redis-server --daemonize yes --port \$REDIS_PORT" >> $REDIS_FILE; && \
+    echo "export SINGULARITYENV_REDIS_PORTS=\$REDIS_PORT" >> $REDIS_FILE; && \
+    chmod +x $REDIS_FILE;
+
+RUN mkdir -p "/usr/local/ilab" /opt/DgStereo && \
+    # EVHR System Dependencies
+    wget "https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/2.7.0/StereoPipeline-2.7.0-2020-07-29-x86_64-Linux.tar.bz2" -O /opt/StereoPipeline.tar.bz2 && \
+    cd /opt && \
+    tar -jxf StereoPipeline.tar.bz2 && \
+    rm /opt/StereoPipeline.tar.bz2 && \
+    mv /opt/StereoPipeline* /opt/StereoPipeline && \
+    pip install redis && \
+    pip install celery[redis] && \
+    # EVHR DEM Dependencies
+    git clone https://github.com/nasa-nccs-hpda/evhr-dgtools.git /opt/DgStereo/dgtools && \
+    git clone https://github.com/nasa-nccs-hpda/evhr-imview.git /opt/DgStereo/imview && \
+    git clone https://github.com/nasa-nccs-hpda/evhr-pygeotools.git /opt/DgStereo/pygeotools && \
+    git clone https://github.com/nasa-nccs-hpda/evhr-dem.git /opt/DgStereo/evhr && \
+    printf "crane103\ncrane104\ncrane105\ncrane106\n\n" > /opt/DgStereo/nodeList.txt && \
+    chmod 755 -R /opt/* && \
+    pip install 2to3 psycopg2-binary && \
+    2to3 -w /opt/DgStereo && \
+    chmod +wx /lib/x86_64-linux-gnu/libc.so.6 && \
+    git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/core.git \
+        /usr/local/ilab/core && \
+    git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/evhr.git \
+        /usr/local/ilab/evhr && \
+    chmod a+rwx -R /usr/local/ilab/* && \
+    # SR-Lite Git Dependencies
+    pip install plotnine==0.10.1 && \
+    pip install --upgrade pylr2 && \
+    git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/srlite.git \
+        /usr/local/ilab/srlite && \
+    chmod a+rwx -R /usr/local/ilab/* && \
+    # CCDC System Dependencies
+    git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/pyCCDC.git \
+        /usr/local/ilab/pyCCDC
+
+#%runscript
+#    /bin/bash -c "source /etc/profile.d/redis_server.sh"
+#    exec "$@"
+#    /bin/bash -c "pkill redis-server"
+#
+#%startscript
+#    #/bin/bash -c "export REDIS_PORT=6379"
+#    exec /bin/bash -c "source /etc/profile.d/redis_server.sh"
+#
+#    # --------------------------------------------------------------------------
+#
+
+HEALTHCHECK NONE
+ENTRYPOINT []
+CMD ["/bin/bash"]
diff --git a/requirements/Dockerfile.cpu b/requirements/Dockerfile.cpu
@@ -19,164 +19,26 @@ ENV C_INCLUDE_PATH="/usr/include/gdal"
 ENV PROJ_LIB="/usr/share/proj"
 ENV PROJ_DATA="/usr/share/proj"
 ENV SM_FRAMEWORK="tf.keras"
-ENV PYTHONPATH="/usr/local/lib/python3.8/dist-packages"
+#ENV PYTHONPATH="/usr/local/lib/python3.8/dist-packages"
 ENV PYTHONPATH="$PYTHONPATH:/usr/local/ilab"
 
 ENV PYTHONPATH="$PYTHONPATH:/usr/local/ilab/vhr-cloudmask:/usr/local/ilab/evhr:/usr/local/ilab/core:/usr/local/ilab/srlite/srlite:/usr/local/ilab/srlite"
 ENV PYTHONPATH="$PYTHONPATH:/opt/DgStereo/pygeotools:/opt/DgStereo/dgtools"
-ENV PYTHONPATH="$PYTHONPATH:/opt/DgStereo/pygeotools:/opt/DgStereo/dgtools"
 ENV PATH="$PATH:/opt/DgStereo/dgtools:/opt/DgStereo/pygeotools:/opt/DgStereo/dgtools/dgtools:/opt/StereoPipeline/bin:/opt/DgStereo/evhr"
-ENV REDIS_PORT="6379"
-
-# System dependencies
-RUN apt-get update && apt-get -y dist-upgrade && \
-    apt-get -y install \
-        build-essential \
-        # EVHR system dependencies
-        parallel \
-        libdatetime-perl \
-        gawk \
-        util-linux \
-        bc \
-        # SRLite system dependencies
-        python3-tk \
-        tk-dev \
-        libgdbm-dev \
-        libc6-dev \
-        libbz2-dev \
-        libffi-dev \
-        zlib1g-dev \
-        liblzma-dev \
-        libgirepository1.0-dev \
-        libcairo2-dev \
-        pkg-config \
-        python3-dev \
-        gir1.2-gtk-3.0 && \
-    apt-get -y autoremove && \
-    rm -rf /var/cache/apt /var/lib/apt/lists/*
 
 RUN PROJECT_PATH="/usr/local/ilab" && \
     REDIS_FILE="/etc/profile.d/redis_server.sh" && \
     echo $PROJECT_PATH && \
     mkdir -p $PROJECT_PATH && \
-    #-------------------------------------------------------------------------------
-    # System Dependencies
-    #-------------------------------------------------------------------------------
-    apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         python3-pip python3-dev wget vim curl git procps gcc g++ bzip2 libssl-dev \
         libsqlite3-dev libx11-dev libgeos++-dev libproj-dev
     DEBIAN_FRONTEND=noninteractive apt-get -y autoremove && \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /var/cache/apt
 
-    #-------------------------------------------------------------------------------
-    # Python Stack
-    #-------------------------------------------------------------------------------
-RUN pip install pipx && \
-     pipx install --upgrade pip && \
-     pip install awscli s3fs && \
-     pip install celery[redis] && \
-     pip install certifi && \
-     pip install cryptography && \
-     pip install Cython && \
-     pip install dask && \
-     pip install decorator && \
-     pip install dill && \
-     pip install distributed && \
-     pip install docutils && \
-     pip install fiona && \
-     pip install flower && \
-     pip install geopandas && \
-     pip install h5py && \
-     pip install imageio && \
-     pip install ipykernel && \
-     pip install ipython && \
-     pip install ipython-genutils && \
-     pip install ipywidgets && \
-     pip install matplotlib && \
-     pip install netcdf4 && \
-     pip install networkx && \
-     pip install notebook && \
-     pip install numba && \
-     pip install numexpr && \
-     pip install numpy && \
-     pip install pandas && \
-     pip install pyhdf && \
-     pip install pyproj && \
-     pip install Pysal && \
-     pip install PyYAML && \
-     pip install rasterio && \
-     pip install redis && \
-     pip install requests && \
-     pip install rioxarray && \
-     pip install scikit-image && \
-     pip install scikit-learn && \
-     pip install scipy && \
-     pip install seaborn && \
-     pip install shapely && \
-     pip install cartopy && \
-     pip install xarray && \
-     pip install urllib3 && \
-     pip install zarr && \
-     pip install earthengine-api && \
-     pip install geemap && \
-     pip install geedim && \
-     pip install vhr-cloudmask && \
-     pip cache purge
-
-#    # Add redis-server binary to /usr/local/bin
-RUN export pyVer=`python --version | awk -F' ' '{print $2}' | awk -F'.' '{print $1"."$2}'` && \
-    ln -sf /usr/local/lib/python${pyVer}/dist-packages/redis_server/bin/redis-server /usr/local/bin/redis-server && \
-    echo "redis-server --daemonize yes --port \$REDIS_PORT" >> $REDIS_FILE; && \
-    echo "export SINGULARITYENV_REDIS_PORTS=\$REDIS_PORT" >> $REDIS_FILE; && \
-    chmod +x $REDIS_FILE;
-
-RUN mkdir -p "/usr/local/ilab" /opt/DgStereo && \
-    # EVHR System Dependencies
-    wget "https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/2.7.0/StereoPipeline-2.7.0-2020-07-29-x86_64-Linux.tar.bz2" -O /opt/StereoPipeline.tar.bz2 && \
-    cd /opt && \
-    tar -jxf StereoPipeline.tar.bz2 && \
-    rm /opt/StereoPipeline.tar.bz2 && \
-    mv /opt/StereoPipeline* /opt/StereoPipeline && \
-    pip install redis && \
-    pip install celery[redis] && \
-    # EVHR DEM Dependencies
-    git clone https://github.com/nasa-nccs-hpda/evhr-dgtools.git /opt/DgStereo/dgtools && \
-    git clone https://github.com/nasa-nccs-hpda/evhr-imview.git /opt/DgStereo/imview && \
-    git clone https://github.com/nasa-nccs-hpda/evhr-pygeotools.git /opt/DgStereo/pygeotools && \
-    git clone https://github.com/nasa-nccs-hpda/evhr-dem.git /opt/DgStereo/evhr && \
-    printf "crane103\ncrane104\ncrane105\ncrane106\n\n" > /opt/DgStereo/nodeList.txt && \
-    chmod 755 -R /opt/* && \
-    pip install 2to3 psycopg2-binary && \
-    2to3 -w /opt/DgStereo && \
-    chmod +wx /lib/x86_64-linux-gnu/libc.so.6 && \
-    git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/core.git \
-        /usr/local/ilab/core && \
-    git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/evhr.git \
-        /usr/local/ilab/evhr && \
-    chmod a+rwx -R /usr/local/ilab/* && \
-    # SR-Lite Git Dependencies
-    pip install plotnine==0.10.1 && \
-    pip install --upgrade pylr2 && \
-    git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/srlite.git \
-        /usr/local/ilab/srlite && \
-    chmod a+rwx -R /usr/local/ilab/* && \
-    # CCDC System Dependencies
-    git clone --single-branch --branch main https://github.com/nasa-nccs-hpda/pyCCDC.git \
-        /usr/local/ilab/pyCCDC
-
-#%runscript
-#    /bin/bash -c "source /etc/profile.d/redis_server.sh"
-#    exec "$@"
-#    /bin/bash -c "pkill redis-server"
-#
-#%startscript
-#    #/bin/bash -c "export REDIS_PORT=6379"
-#    exec /bin/bash -c "source /etc/profile.d/redis_server.sh"
-#
-#    # --------------------------------------------------------------------------
-#
 
 HEALTHCHECK NONE
 ENTRYPOINT []