diff --git a/.github/workflows/docker/compose/dataprep-compose-cd.yaml b/.github/workflows/docker/compose/dataprep-compose-cd.yaml index c7c2089a3..7bf008732 100644 --- a/.github/workflows/docker/compose/dataprep-compose-cd.yaml +++ b/.github/workflows/docker/compose/dataprep-compose-cd.yaml @@ -23,10 +23,6 @@ services: build: dockerfile: comps/dataprep/multimodal/redis/langchain/Dockerfile image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest} - dataprep-vdms: - build: - dockerfile: comps/dataprep/vdms/multimodal_langchain/Dockerfile - image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest} dataprep-vdmsxx: #need to rename build: dockerfile: comps/dataprep/vdms/langchain/Dockerfile diff --git a/.github/workflows/docker/compose/dataprep-compose.yaml b/.github/workflows/docker/compose/dataprep-compose.yaml index b697af133..bb5f08218 100644 --- a/.github/workflows/docker/compose/dataprep-compose.yaml +++ b/.github/workflows/docker/compose/dataprep-compose.yaml @@ -17,3 +17,7 @@ services: build: dockerfile: comps/dataprep/redis/langchain_ray/Dockerfile image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest} + dataprep-vdms: + build: + dockerfile: comps/dataprep/vdms/multimodal_langchain/Dockerfile + image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest} diff --git a/.github/workflows/docker/compose/embeddings-compose-cd.yaml b/.github/workflows/docker/compose/embeddings-compose-cd.yaml index 8abb89aa3..422038466 100644 --- a/.github/workflows/docker/compose/embeddings-compose-cd.yaml +++ b/.github/workflows/docker/compose/embeddings-compose-cd.yaml @@ -14,10 +14,6 @@ services: build: dockerfile: comps/embeddings/tei/llama_index/Dockerfile image: ${REGISTRY:-opea}/embedding-tei-llama-index:${TAG:-latest} - embedding-multimodal: - build: - dockerfile: comps/embeddings/multimodal_clip/Dockerfile - image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest} bridgetower-embedder: build: dockerfile: comps/embeddings/multimodal/bridgetower/Dockerfile diff --git a/.github/workflows/docker/compose/embeddings-compose.yaml b/.github/workflows/docker/compose/embeddings-compose.yaml index a922f0611..a97c8460f 100644 --- a/.github/workflows/docker/compose/embeddings-compose.yaml +++ b/.github/workflows/docker/compose/embeddings-compose.yaml @@ -8,3 +8,7 @@ services: build: dockerfile: comps/embeddings/tei/langchain/Dockerfile image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest} + embedding-multimodal: + build: + dockerfile: comps/embeddings/multimodal_clip/Dockerfile + image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest} diff --git a/.github/workflows/docker/compose/lvms-compose-cd.yaml b/.github/workflows/docker/compose/lvms-compose-cd.yaml index c5a72df21..a1790cf93 100644 --- a/.github/workflows/docker/compose/lvms-compose-cd.yaml +++ b/.github/workflows/docker/compose/lvms-compose-cd.yaml @@ -17,11 +17,3 @@ services: build: dockerfile: comps/lvms/llava/dependency/Dockerfile.intel_hpu image: ${REGISTRY:-opea}/llava-hpu:${TAG:-latest} - lvm-video-llama: - build: - dockerfile: comps/lvms/video-llama/Dockerfile - image: ${REGISTRY:-opea}/lvm-video-llama:${TAG:-latest} - video-llama-lvm-server: - build: - dockerfile: comps/lvms/video-llama/dependency/Dockerfile - image: ${REGISTRY:-opea}/video-llama-lvm-server:${TAG:-latest} diff --git a/.github/workflows/docker/compose/lvms-compose.yaml b/.github/workflows/docker/compose/lvms-compose.yaml index 5df2c5182..d49c3f368 100644 --- a/.github/workflows/docker/compose/lvms-compose.yaml +++ b/.github/workflows/docker/compose/lvms-compose.yaml @@ -7,3 +7,11 @@ services: build: dockerfile: comps/lvms/tgi-llava/Dockerfile image: ${REGISTRY:-opea}/lvm-tgi:${TAG:-latest} + lvm-video-llama: + build: + dockerfile: comps/lvms/video-llama/Dockerfile + image: ${REGISTRY:-opea}/lvm-video-llama:${TAG:-latest} + video-llama-lvm-server: + build: + dockerfile: comps/lvms/video-llama/dependency/Dockerfile + image: ${REGISTRY:-opea}/video-llama-lvm-server:${TAG:-latest} diff --git a/.github/workflows/docker/compose/reranks-compose-cd.yaml b/.github/workflows/docker/compose/reranks-compose-cd.yaml index f6180e0ef..3e5e7caab 100644 --- a/.github/workflows/docker/compose/reranks-compose-cd.yaml +++ b/.github/workflows/docker/compose/reranks-compose-cd.yaml @@ -14,7 +14,3 @@ services: build: dockerfile: comps/reranks/mosec/langchain/Dockerfile image: ${REGISTRY:-opea}/reranking-langchain-mosec:${TAG:-latest} - reranking-videoragqna: - build: - dockerfile: comps/reranks/video-rag-qna/Dockerfile - image: ${REGISTRY:-opea}/reranking-videoragqna:${TAG:-latest} diff --git a/.github/workflows/docker/compose/reranks-compose.yaml b/.github/workflows/docker/compose/reranks-compose.yaml index 25c0aeb9b..b11fb8a4a 100644 --- a/.github/workflows/docker/compose/reranks-compose.yaml +++ b/.github/workflows/docker/compose/reranks-compose.yaml @@ -8,3 +8,7 @@ services: build: dockerfile: comps/reranks/tei/Dockerfile image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest} + reranking-videoragqna: + build: + dockerfile: comps/reranks/video-rag-qna/Dockerfile + image: ${REGISTRY:-opea}/reranking-videoragqna:${TAG:-latest} diff --git a/.github/workflows/docker/compose/retrievers-compose-cd.yaml b/.github/workflows/docker/compose/retrievers-compose-cd.yaml index fdf128b1e..f9230412d 100644 --- a/.github/workflows/docker/compose/retrievers-compose-cd.yaml +++ b/.github/workflows/docker/compose/retrievers-compose-cd.yaml @@ -27,7 +27,3 @@ services: build: dockerfile: comps/retrievers/multimodal/redis/langchain/Dockerfile image: ${REGISTRY:-opea}/multimodal-retriever-redis:${TAG:-latest} - retriever-vdms: - build: - dockerfile: comps/retrievers/vdms/langchain/Dockerfile - image: ${REGISTRY:-opea}/retriever-vdms:${TAG:-latest} diff --git a/.github/workflows/docker/compose/retrievers-compose.yaml b/.github/workflows/docker/compose/retrievers-compose.yaml index db1c19530..80680a61f 100644 --- a/.github/workflows/docker/compose/retrievers-compose.yaml +++ b/.github/workflows/docker/compose/retrievers-compose.yaml @@ -11,3 +11,7 @@ services: build: dockerfile: comps/retrievers/qdrant/haystack/Dockerfile image: ${REGISTRY:-opea}/retriever-qdrant:${TAG:-latest} + retriever-vdms: + build: + dockerfile: comps/retrievers/vdms/langchain/Dockerfile + image: ${REGISTRY:-opea}/retriever-vdms:${TAG:-latest} diff --git a/.github/workflows/docker/compose/tts-compose-cd.yaml b/.github/workflows/docker/compose/tts-compose-cd.yaml deleted file mode 100644 index 8adad51e0..000000000 --- a/.github/workflows/docker/compose/tts-compose-cd.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -# this file should be run in the root of the repo -# images used by GenAIExamples: reranking-tei -services: - tts: #need to rename - build: - dockerfile: comps/tts/Dockerfile - image: ${REGISTRY:-opea}/tts:${TAG:-latest} diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index ce5d6501f..125a3e80b 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -56,66 +56,60 @@ jobs: exit 1 fi - # - name: Check all Dockerfile mode and if it exists in yaml - # if: always() - # run: | - # set -e - # shopt -s globstar - # no_add="FALSE" - # cd ${{github.workspace}} - # Dockerfiles=$(find ./comps -name '*Dockerfile*'|sed 's/^\.\///') - # if [ -n "$Dockerfiles" ]; then - # for Dockerfile in $Dockerfiles; do - # service=$(echo "$Dockerfile" | awk -F '/' '{print $2}') - # if grep -q "$Dockerfile" ../GenAIExamples/**/*build.yaml*; then - # mode="" #CI - # else - # mode="-cd" #CD - # fi - # - # yaml_file=${{github.workspace}}/.github/workflows/docker/compose/$service'-compose'$mode'.yaml' - # if [ -e "$yaml_file" ]; then - # # check service yaml if exist - # if ! grep -q "$Dockerfile" "$yaml_file"; then - # echo "AR: Update $Dockerfile to .github/workflows/docker/compose/"$service"-compose"$mode".yaml." - # no_add="TRUE" - # fi - # else - # echo "AR: Create a new compose file .github/workflows/docker/compose/"$service"-compose"$mode".yaml for $Dockerfile and fill it in." - # no_add="TRUE" - # fi - # done - # fi - # - # if [[ "$no_add" == "TRUE" ]]; then - # exit 1 - # fi - # - # - name: Check for the exist of Dockerfile in all yamls. - # if: always() - # run: | - # shopt -s globstar - # no_exist="FALSE" - # cd ${{github.workspace}} - # yamls=$(find .github/workflows/docker/compose/ -name '*.yaml') - # if [ -n "$yamls" ]; then - # for yaml in $yamls; do - # dockerfiles=$(grep 'dockerfile:' "$yaml"|sed 's/dockerfile: //') - # for dockerfile in $dockerfiles; do - # if [[ "$dockerfile" == *"comps/"* ]]; then - # if ! [ -e "$dockerfile" ]; then - # # cat "$yaml" - # echo "AR: The dockerfile path "$dockerfile" in "$yaml" does not exist, remove or update it." - # no_exist="TRUE" - # fi - # fi - # done - # done - # fi - # - # if [[ "$no_exist" == "TRUE" ]]; then - # exit 1 - # fi + - name: Check Dockerfile path included in image build yaml + if: always() + run: | + set -e + shopt -s globstar + no_add="FALSE" + cd ${{github.workspace}} + Dockerfiles=$(find ./comps -name '*Dockerfile*'|sed 's/^\.\///') + if [ -n "$Dockerfiles" ]; then + for Dockerfile in $Dockerfiles; do + service=$(echo "$Dockerfile" | awk -F '/' '{print $2}') + if grep -q "$Dockerfile" ../GenAIExamples/**/*build.yaml*; then + mode="" #CI + else + mode="-cd" #CD + fi + + yaml_file=${{github.workspace}}/.github/workflows/docker/compose/"$service"-compose + if ! grep -q "$Dockerfile" "$yaml_file"*yaml; then + echo "AR: Update $Dockerfile to .github/workflows/docker/compose/"$service"-compose"$mode".yaml." + no_add="TRUE" + fi + done + fi + + if [[ "$no_add" == "TRUE" ]]; then + exit 1 + fi + + - name: Check Dockerfile inside image build yaml exist in code + if: always() + run: | + shopt -s globstar + no_exist="FALSE" + cd ${{github.workspace}} + yamls=$(find .github/workflows/docker/compose/ -name '*.yaml') + if [ -n "$yamls" ]; then + for yaml in $yamls; do + dockerfiles=$(grep 'dockerfile:' "$yaml"|sed 's/dockerfile: //') + for dockerfile in $dockerfiles; do + if [[ "$dockerfile" == *"comps/"* ]]; then + if ! [ -e "$dockerfile" ]; then + # cat "$yaml" + echo "AR: The dockerfile path "$dockerfile" in "$yaml" does not exist, remove or update it." + no_exist="TRUE" + fi + fi + done + done + fi + + if [[ "$no_exist" == "TRUE" ]]; then + exit 1 + fi Dockerfile-path-change-detection-in-GenAIExamples: runs-on: ubuntu-latest