Skip to content

Commit

Permalink
use more appropriate name for dpk wheel
Browse files Browse the repository at this point in the history
Signed-off-by: Maroun Touma <touma@us.ibm.com>
  • Loading branch information
touma-I committed Oct 26, 2024
1 parent 88438d1 commit 1b84023
Show file tree
Hide file tree
Showing 55 changed files with 110 additions and 110 deletions.
8 changes: 4 additions & 4 deletions .make.defaults
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ __check_defined = \
--platform $(DOCKER_PLATFORM) \
--build-arg EXTRA_INDEX_URL=$(EXTRA_INDEX_URL) \
--build-arg BASE_IMAGE=$(BASE_IMAGE) \
--build-arg WHEEL_FILE_NAME=$(WHEEL_FILE_NAME) \
--build-arg DPK_WHEEL_FILE_NAME=$(DPK_WHEEL_FILE_NAME) \
--build-arg BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') \
--build-arg GIT_COMMIT=$(shell git log -1 --format=%h) .
$(DOCKER) tag $(DOCKER_LOCAL_IMAGE) $(DOCKER_REMOTE_IMAGE)
Expand Down Expand Up @@ -270,7 +270,7 @@ __check_defined = \
@# Help: Build the Python $(DOCKER_LOCAL_IMAGE) using the the wheel file for the library
@$(eval LIB_WHEEL_FILE := $(shell find data-processing-dist/*.whl))
$(eval LIB_WHEEL_FILE := $(shell basename $(LIB_WHEEL_FILE)))
$(MAKE) DOCKER_IMAGE=$(DOCKER_LOCAL_IMAGE) WHEEL_FILE_NAME=$(LIB_WHEEL_FILE) .defaults.image
$(MAKE) DOCKER_IMAGE=$(DOCKER_LOCAL_IMAGE) DPK_WHEEL_FILE_NAME=$(LIB_WHEEL_FILE) .defaults.image
-rm -rf data-processing-dist

# Build an image using the local Dockerfile and make the data-processing-lib/ray
Expand Down Expand Up @@ -306,7 +306,7 @@ __check_defined = \
if [ -e ../python ]; then \
$(MAKE) LIB_PATH=../python LIB_NAME=python-transform .defaults.copy-lib; \
fi
$(MAKE) DOCKER_IMAGE=$(DOCKER_LOCAL_IMAGE) WHEEL_FILE_NAME=$(LIB_WHEEL_FILE) .defaults.image
$(MAKE) DOCKER_IMAGE=$(DOCKER_LOCAL_IMAGE) DPK_WHEEL_FILE_NAME=$(LIB_WHEEL_FILE) .defaults.image
-rm -rf python-transform
-rm -rf data-processing-dist

Expand Down Expand Up @@ -345,7 +345,7 @@ __check_defined = \
if [ -e ../python ]; then \
$(MAKE) LIB_PATH=../python LIB_NAME=python-transform .defaults.copy-lib; \
fi
$(MAKE) DOCKER_IMAGE=$(DOCKER_LOCAL_IMAGE) BASE_IMAGE=$(DOCKER_SPARK_BASE_IMAGE) WHEEL_FILE_NAME=$(LIB_WHEEL_FILE) .defaults.image
$(MAKE) DOCKER_IMAGE=$(DOCKER_LOCAL_IMAGE) BASE_IMAGE=$(DOCKER_SPARK_BASE_IMAGE) DPK_WHEEL_FILE_NAME=$(LIB_WHEEL_FILE) .defaults.image
-rm -rf python-transform
-rm -rf data-processing-dist

Expand Down
4 changes: 2 additions & 2 deletions kfp/kfp_ray_components/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ FROM ${BASE_IMAGE}
COPY requirements.txt requirements.txt
RUN pip install --no-cache-dir -r requirements.txt

ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

COPY --chown=ray:users shared_workflow_support_lib shared_workflow_support_lib/
RUN cd shared_workflow_support_lib && pip install --no-cache-dir -e .
Expand Down
2 changes: 1 addition & 1 deletion kfp/kfp_ray_components/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ endif
$(eval LIB_WHEEL_FILE := $(shell basename $(LIB_WHEEL_FILE)))
$(MAKE) .defaults.copy-lib LIB_PATH=$(REPOROOT)/kfp/kfp_support_lib/shared_workflow_support LIB_NAME=shared_workflow_support_lib
$(MAKE) .defaults.copy-lib LIB_PATH=$(REPOROOT)/kfp/kfp_support_lib/$(WORKFLOW_SUPPORT_LIB) LIB_NAME=workflow_support_lib
$(MAKE) WHEEL_FILE_NAME=$(LIB_WHEEL_FILE) .defaults.image
$(MAKE) DPK_WHEEL_FILE_NAME=$(LIB_WHEEL_FILE) .defaults.image
rm -rf data-processing-dist
rm -rf shared_workflow_support_lib
rm -rf workflow_support_lib
Expand Down
4 changes: 2 additions & 2 deletions tools/ingest2parquet/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ FROM ${BASE_IMAGE}

# install pytest
RUN pip install --no-cache-dir pytest
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

COPY requirements.txt requirements.txt
RUN pip install --no-cache-dir -r requirements.txt
Expand Down
4 changes: 2 additions & 2 deletions transforms/code/code2parquet/python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ RUN pip install --no-cache-dir pytest
RUN useradd -ms /bin/bash dpk
USER dpk
WORKDIR /home/dpk
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=dpk:root data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}

# END OF STEPS destined for a data-prep-kit base image

Expand Down
4 changes: 2 additions & 2 deletions transforms/code/code2parquet/ray/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ RUN pip install --upgrade --no-cache-dir pip
# install pytest
RUN pip install --no-cache-dir pytest

ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

COPY --chown=ray:users python-transform/ python-transform/
RUN cd python-transform && pip install --no-cache-dir -e .
Expand Down
4 changes: 2 additions & 2 deletions transforms/code/code_profiler/python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ RUN pip install --no-cache-dir pytest
RUN useradd -ms /bin/bash dpk
USER dpk
WORKDIR /home/dpk
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=dpk:root data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}

COPY --chown=dpk:root src/ src/
COPY --chown=dpk:root pyproject.toml pyproject.toml
Expand Down
4 changes: 2 additions & 2 deletions transforms/code/code_profiler/ray/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ RUN pip install --upgrade --no-cache-dir pip
# install pytest
RUN pip install --no-cache-dir pytest

ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

COPY --chown=ray:users python-transform/ python-transform/
RUN cd python-transform && pip install --no-cache-dir -e .
Expand Down
4 changes: 2 additions & 2 deletions transforms/code/code_quality/python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ RUN pip install --no-cache-dir pytest
RUN useradd -ms /bin/bash dpk
USER dpk
WORKDIR /home/dpk
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=dpk:root data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}

# END OF STEPS destined for a data-prep-kit base image

Expand Down
4 changes: 2 additions & 2 deletions transforms/code/code_quality/ray/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ RUN pip install --no-cache-dir pytest
USER root
RUN mkdir -p /home/ray/.cache && chmod -R 777 /home/ray/.cache
USER ray
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

COPY --chown=ray:users python-transform/ python-transform/
RUN cd python-transform && pip install --no-cache-dir -e .
Expand Down
4 changes: 2 additions & 2 deletions transforms/code/header_cleanser/python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@ WORKDIR /home/dpk

# Create directories to mount volumnes for processing data outside of the image.
# RUN mkdir input && mkdir output
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=dpk:root data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}

# END OF STEPS destined for a data-prep-kit base image

Expand Down
4 changes: 2 additions & 2 deletions transforms/code/header_cleanser/ray/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ FROM docker.io/rayproject/ray:2.24.0-py310
# install pytest
RUN pip install --no-cache-dir pytest

ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

COPY --chown=ray:users python-transform/ python-transform
RUN cd python-transform && pip install --no-cache-dir -e .
Expand Down
4 changes: 2 additions & 2 deletions transforms/code/license_select/python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@ RUN pip install --no-cache-dir pytest
RUN useradd -ms /bin/bash dpk
USER dpk
WORKDIR /home/dpk
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=dpk:root data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}

# END OF STEPS destined for a data-prep-kit base image

Expand Down
4 changes: 2 additions & 2 deletions transforms/code/license_select/ray/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ RUN pip install --upgrade --no-cache-dir pip

# install pytest
RUN pip install --no-cache-dir pytest
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

COPY --chown=ray:users python-transform/ python-transform/
RUN cd python-transform && pip install --no-cache-dir -e .
Expand Down
4 changes: 2 additions & 2 deletions transforms/code/malware/python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@ COPY --chown=dpk:root --from=clamav-local /var/lib/clamav/ /var/lib/clamav/
COPY --chown=dpk:root --from=clamav-local /etc/clamav/clamd.conf /etc/clamav/clamd.conf
COPY --chown=dpk:root --from=clamav-local /var/log/clamav/clamav.log /var/log/clamav/clamav.log
COPY --chown=dpk:root --from=clamav-local /var/run/clamav /var/run/clamav
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=dpk:root data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}

COPY --chown=dpk:root src/ src/
COPY --chown=dpk:root pyproject.toml pyproject.toml
Expand Down
4 changes: 2 additions & 2 deletions transforms/code/malware/ray/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ COPY --from=clamav-local --chown=ray:0 /etc/clamav/clamd.conf /etc/clamav/clamd.
COPY --from=clamav-local --chown=ray:0 /var/log/clamav /var/log/clamav
COPY --from=clamav-local --chown=ray:0 /var/run/clamav /var/run/clamav

ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

COPY --chown=ray:users python-transform/ python-transform/
RUN cd python-transform && pip install --no-cache-dir -e .
Expand Down
4 changes: 2 additions & 2 deletions transforms/code/proglang_select/python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ RUN pip install --no-cache-dir pytest
RUN useradd -ms /bin/bash dpk
USER dpk
WORKDIR /home/dpk
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=dpk:root data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}

# END OF STEPS destined for a data-prep-kit base image

Expand Down
4 changes: 2 additions & 2 deletions transforms/code/proglang_select/ray/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ RUN pip install --upgrade --no-cache-dir pip

# install pytest
RUN pip install --no-cache-dir pytest
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

COPY --chown=ray:users python-transform/ python-transform/
RUN cd python-transform && pip install --no-cache-dir -e .
Expand Down
4 changes: 2 additions & 2 deletions transforms/code/repo_level_ordering/ray/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ RUN pip install --upgrade --no-cache-dir pip

# install pytest
RUN pip install --no-cache-dir pytest
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

COPY --chown=ray:users src/ src/
COPY --chown=ray:users pyproject.toml pyproject.toml
Expand Down
4 changes: 2 additions & 2 deletions transforms/language/doc_chunk/python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ USER dpk
WORKDIR /home/dpk

ARG PIP_INSTALL_EXTRA_ARGS
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=dpk:root data-processing-dist/ data-processing-dist/
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}

# END OF STEPS destined for a data-prep-kit base image

Expand Down
4 changes: 2 additions & 2 deletions transforms/language/doc_chunk/ray/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ FROM ${BASE_IMAGE}
RUN pip install --no-cache-dir pytest

ARG PIP_INSTALL_EXTRA_ARGS
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

## Copy the python version of the tansform
COPY --chown=ray:users python-transform/ python-transform/
Expand Down
4 changes: 2 additions & 2 deletions transforms/language/doc_quality/python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ RUN useradd -ms /bin/bash dpk
USER dpk
WORKDIR /home/dpk

ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=dpk:root data-processing-dist/ data-processing-dist/
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}

# END OF STEPS destined for a data-prep-kit base image

Expand Down
4 changes: 2 additions & 2 deletions transforms/language/doc_quality/ray/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ RUN pip install --upgrade --no-cache-dir pip

# install pytest
RUN pip install --no-cache-dir pytest
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

## Copy the python version of the tansform
COPY --chown=ray:users python-transform/ python-transform/
Expand Down
4 changes: 2 additions & 2 deletions transforms/language/html2parquet/python/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ RUN useradd -ms /bin/bash dpk
USER dpk
WORKDIR /home/dpk

ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=dpk:root data-processing-dist/ data-processing-dist/
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}

# END OF STEPS destined for a data-prep-kit base image

Expand Down
4 changes: 2 additions & 2 deletions transforms/language/html2parquet/ray/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ RUN pip install --upgrade --no-cache-dir pip

# install pytest
RUN pip install --no-cache-dir pytest
ARG WHEEL_FILE_NAME
ARG DPK_WHEEL_FILE_NAME

# Copy and install data processing libraries
# These are expected to be placed in the docker context before this is run (see the make image).
COPY --chown=ray:users data-processing-dist data-processing-dist
RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
RUN pip install data-processing-dist/${DPK_WHEEL_FILE_NAME}[ray]

## Copy the python version of the tansform
COPY --chown=ray:users python-transform/ python-transform/
Expand Down
Loading

0 comments on commit 1b84023

Please sign in to comment.