Merge pull request IBM#893 from IBM/fdedup-simplify

FDedup Refactored as its own dpk_ module

transforms/universal/fdedup/python/Dockerfile → transforms/universal/fdedup/Dockerfile.python

@@ -2,34 +2,21 @@ FROM docker.io/python:3.10.14-slim-bullseye
 
 RUN pip install --upgrade --no-cache-dir pip 
 
-# install pytest
-RUN pip install --no-cache-dir pytest
-ARG DPK_WHEEL_FILE_NAME
-
 # Create a user and use it to run the transform
 RUN useradd -ms /bin/bash dpk
 USER dpk
 WORKDIR /home/dpk
+ARG DPK_WHEEL_FILE_NAME
 
 # Copy and install data processing libraries 
 # These are expected to be placed in the docker context before this is run (see the make image).
 COPY --chown=dpk:root data-processing-dist data-processing-dist
-RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}
+RUN  pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}
 
-COPY --chown=dpk:root src/ src/
-COPY --chown=dpk:root pyproject.toml pyproject.toml
-COPY --chown=dpk:root README.md README.md
+COPY --chown=dpk:root dpk_fdedup/ dpk_fdedup/
 COPY --chown=dpk:root requirements.txt requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
-RUN pip install --no-cache-dir -e .
-
-# copy source data
-COPY ./src/fdedup_transform_python.py fdedup_transform_python.py
-COPY ./src/fdedup_transform_python.py local/
-
-# copy test
-COPY test/ test/
-COPY test-data/ test-data/
 
 # Set environment
 ENV PYTHONPATH /home/dpk
@@ -38,4 +25,4 @@ ENV PYTHONPATH /home/dpk
 ARG BUILD_DATE
 ARG GIT_COMMIT
 LABEL build-date=$BUILD_DATE
-LABEL git-commit=$GIT_COMMIT
+LABEL git-commit=$GIT_COMMIT

transforms/universal/fdedup/Dockerfile.ray

@@ -0,0 +1,31 @@
+ARG BASE_IMAGE=docker.io/rayproject/ray:2.36.1-py310
+
+FROM ${BASE_IMAGE}
+
+RUN pip install --upgrade --no-cache-dir pip 
+
+# install pytest
+RUN pip install --no-cache-dir pytest
+ARG DPK_WHEEL_FILE_NAME
+
+# Copy and install data processing libraries 
+# These are expected to be placed in the docker context before this is run (see the make image).
+COPY --chown=ray:users data-processing-dist data-processing-dist
+RUN  pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]
+
+## Copy the python version of the tansform
+COPY --chown=ray:users dpk_fdedup/ dpk_fdedup/
+COPY --chown=ray:users requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+# Grant non-root users the necessary permissions to the ray directory
+RUN chmod 755 /home/ray
+
+# Set environment
+ENV PYTHONPATH /home/ray
+
+# Put these at the end since they seem to upset the docker cache.
+ARG BUILD_DATE
+ARG GIT_COMMIT
+LABEL build-date=$BUILD_DATE
+LABEL git-commit=$GIT_COMMIT

transforms/universal/fdedup/Dockerfile.spark

@@ -0,0 +1,40 @@
+FROM quay.io/dataprep1/data-prep-kit/data-prep-kit-spark-3.5.2:latest
+
+USER root
+# install pytest
+RUN pip install --no-cache-dir pytest
+
+WORKDIR ${SPARK_HOME}/work-dir
+ARG DPK_WHEEL_FILE_NAME
+
+# Copy and install data processing libraries 
+# These are expected to be placed in the docker context before this is run (see the make image).
+COPY --chown=spark:root data-processing-dist data-processing-dist
+RUN  pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[spark]
+
+
+# Install project source
+
+## Copy the python version of the tansform
+COPY --chown=spark:root dpk_fdedup/ dpk_fdedup/
+COPY --chown=spark:root requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+RUN mkdir -p /opt/spark/work-dir/src/templates && \
+    mkdir -p /opt/spark/work-dir/config
+COPY --chown=spark:root spark/deployment/kubernetes/spark-executor-pod-template.yml /opt/spark/work-dir/src/templates/
+COPY --chown=spark:root spark/deployment/kubernetes/spark_profile.yml /opt/spark/work-dir/config/
+
+
+USER spark
+
+# Set environment
+ENV PYTHONPATH=${SPARK_HOME}/work-dir/:${SPARK_HOME}/work-dir/src/:${PYTHONPATH}
+ENV PATH=${SPARK_HOME}/work-dir/.local/bin/:${PATH}
+
+# Put these at the end since they seem to upset the docker cache.
+ARG BUILD_DATE
+ARG GIT_COMMIT
+LABEL build-date=$BUILD_DATE
+LABEL git-commit=$GIT_COMMIT
+

transforms/universal/fdedup/Makefile

@@ -1,79 +1,17 @@
 REPOROOT=../../..
 # Use make help, to see the available rules
-include $(REPOROOT)/.make.defaults
+include $(REPOROOT)/transforms/.make.cicd.targets
 
-setup::
-	@# Help: Recursively make $@ all subdirs 
-	$(MAKE) RULE=$@ .recurse
+#
+# This is intended to be included across the Makefiles provided within
+# a given transform's directory tree,  so must use compatible syntax.
+#
+################################################################################
+# This defines the name of the transform and is used to match against
+# expected files and is used to define the transform's image name. 
+TRANSFORM_NAME=$(shell basename `pwd`)
 
-clean::
-	@# Help: Recursively make $@ all subdirs 
-	$(MAKE) RULE=$@ .recurse
+################################################################################
 
-build::
-	@# Help: Recursively make $@ in subdirs 
-	$(MAKE) RULE=$@ .recurse
-venv::
-	@# Help: Recursively make $@ in subdirs 
-	$(MAKE) RULE=$@ .recurse
-
-image:: 
-	@# Help: Recursively make $@ in all subdirs 
-	@$(MAKE) RULE=$@ .recurse
-
-set-versions:  
-	@# Help: Recursively $@ in all subdirs 
-	@$(MAKE) RULE=$@ .recurse
-
-publish:: 
-	@# Help: Recursively make $@ in all subdirs 
-	@$(MAKE) RULE=$@ .recurse
-
-test-image:: 
-	@# Help: Recursively make $@ in all subdirs 
-	@$(MAKE) RULE=$@ .recurse
-
-test:: 
-	@# Help: Recursively make $@ in all subdirs 
-	@$(MAKE) RULE=$@ .recurse
-
-test-src::
-	@# Help: Recursively make $@ in all subdirs 
-	$(MAKE) RULE=$@ .recurse
-
-kind-load-image::
-	@# Help: Recursively make $@ in all subdirs 
-	$(MAKE) RULE=$@ .recurse
-
-docker-load-image::
-	@# Help: Recursively make $@ in all subdirs
-	$(MAKE) RULE=$@ .recurse
-
-docker-save-image::
-	@# Help: Recursively make $@ in all subdirs 
-	$(MAKE) RULE=$@ .recurse
-
-.PHONY: workflow-venv
-workflow-venv:
-	if [ -e kfp_ray ]; then                 \
-	    $(MAKE) -C kfp_ray workflow-venv;   \
-	fi
-
-.PHONY: workflow-test
-workflow-test:
-	if [ -e kfp_ray ]; then                 \
-	    $(MAKE) -C kfp_ray workflow-test;   \
-	fi
-
-.PHONY: workflow-upload
-workflow-upload:
-	if [ -e kfp_ray ]; then                 \
-	    $(MAKE) -C kfp_ray workflow-upload; \
-	fi
-
-.PHONY: workflow-build
-workflow-build:
-	if [ -e kfp_ray ]; then                 \
-	    $(MAKE) -C  kfp_ray workflow-build; \
-	fi
 
+