Skip to content

Commit c48cd65

Browse files
chensuyuechensuyue
authored
[CICD enhance] ChatQnA run CI with latest base image, group logs in GHA outputs. (#1736)
Signed-off-by: chensuyue <suyue.chen@intel.com>
1 parent d627209 commit c48cd65

22 files changed

+413
-290
lines changed

‎.github/workflows/_run-docker-compose.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,11 @@ jobs:
115115
- name: Clean up Working Directory
116116
run: |
117117
sudo rm -rf ${{github.workspace}}/* || true
118+
119+
# clean up containers use ports
120+
cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 5000 | awk -F' : ' '{print $1}')
121+
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
122+
118123
docker system prune -f
119124
docker rmi $(docker images --filter reference="*/*/*:latest" -q) || true
120125
docker rmi $(docker images --filter reference="*/*:ci" -q) || true
@@ -175,6 +180,11 @@ jobs:
175180
export test_case=${{ matrix.test_case }}
176181
export hardware=${{ inputs.hardware }}
177182
bash ${{ github.workspace }}/.github/workflows/scripts/docker_compose_clean_up.sh "containers"
183+
184+
# clean up containers use ports
185+
cid=$(docker ps --format '{{.Names}} : {{.Ports}}' | grep -v ' : $' | grep -v 5000 | awk -F' : ' '{print $1}')
186+
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
187+
178188
docker system prune -f
179189
docker rmi $(docker images --filter reference="*:5000/*/*" -q) || true
180190

‎.github/workflows/nightly-docker-build-publish.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ jobs:
6161

6262
publish:
6363
needs: [get-build-matrix, get-image-list, build-and-test]
64-
if: ${{ needs.get-image-list.outputs.matrix != '' }}
64+
if: always() && ${{ needs.get-image-list.outputs.matrix != '' }}
6565
strategy:
6666
matrix:
6767
image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}

‎ChatQnA/Dockerfile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4+
ARG IMAGE_REPO=opea
45
ARG BASE_TAG=latest
5-
FROM opea/comps-base:$BASE_TAG
6+
FROM $IMAGE_REPO/comps-base:$BASE_TAG
67

78
COPY ./chatqna.py $HOME/chatqna.py
89
COPY ./entrypoint.sh $HOME/entrypoint.sh

‎ChatQnA/docker_image_build/build.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ services:
55
chatqna:
66
build:
77
args:
8+
IMAGE_REPO: ${REGISTRY}
9+
BASE_TAG: ${TAG}
810
http_proxy: ${http_proxy}
911
https_proxy: ${https_proxy}
1012
no_proxy: ${no_proxy}

‎ChatQnA/tests/test_compose_faqgen_on_gaudi.sh

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,12 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020
cd $WORKPATH/docker_image_build
3121
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
22+
pushd GenAIComps
23+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
24+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
25+
popd && sleep 1s
3226
git clone https://github.com/HabanaAI/vllm-fork.git && cd vllm-fork
3327
VLLM_VER=$(git describe --tags "$(git rev-list --tags --max-count=1)")
3428
git checkout ${VLLM_VER} &> /dev/null && cd ../
@@ -250,20 +244,35 @@ function stop_docker() {
250244

251245
function main() {
252246

247+
echo "::group::start_docker"
253248
stop_docker
249+
echo "::endgroup::"
250+
251+
echo "::group::build_docker_images"
254252
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
255-
start_time=$(date +%s)
253+
echo "::endgroup::"
254+
255+
echo "::group::start_services"
256256
start_services
257-
end_time=$(date +%s)
258-
duration=$((end_time-start_time))
259-
echo "Mega service start duration is $duration s"
257+
echo "::endgroup::"
260258

259+
echo "::group::validate_microservices"
261260
validate_microservices
261+
echo "::endgroup::"
262+
263+
echo "::group::validate_megaservice"
262264
validate_megaservice
265+
echo "::endgroup::"
266+
267+
echo "::group::validate_frontend"
263268
validate_frontend
269+
echo "::endgroup::"
264270

271+
echo "::group::stop_docker"
265272
stop_docker
266-
echo y | docker system prune
273+
echo "::endgroup::"
274+
275+
docker system prune -f
267276

268277
}
269278

‎ChatQnA/tests/test_compose_faqgen_on_rocm.sh

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -58,19 +58,12 @@ export PATH="~/miniconda3/bin:$PATH"
5858

5959
function build_docker_images() {
6060
opea_branch=${opea_branch:-"main"}
61-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
62-
if [[ "${opea_branch}" != "main" ]]; then
63-
cd $WORKPATH
64-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
65-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
66-
find . -type f -name "Dockerfile*" | while read -r file; do
67-
echo "Processing file: $file"
68-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
69-
done
70-
fi
71-
7261
cd $WORKPATH/docker_image_build
7362
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
63+
pushd GenAIComps
64+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
65+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
66+
popd && sleep 1s
7467

7568
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
7669
service_list="chatqna chatqna-ui dataprep retriever llm-faqgen nginx"
@@ -247,23 +240,35 @@ function stop_docker() {
247240

248241
function main() {
249242

243+
echo "::group::start_docker"
250244
stop_docker
245+
echo "::endgroup::"
246+
247+
echo "::group::build_docker_images"
251248
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
252-
start_time=$(date +%s)
249+
echo "::endgroup::"
250+
251+
echo "::group::start_services"
253252
start_services
254-
end_time=$(date +%s)
255-
duration=$((end_time-start_time))
256-
echo "Mega service start duration is $duration s" && sleep 1s
253+
echo "::endgroup::"
257254

255+
echo "::group::validate_microservices"
258256
validate_microservices
259-
echo "==== microservices validated ===="
257+
echo "::endgroup::"
258+
259+
echo "::group::validate_megaservice"
260260
validate_megaservice
261-
echo "==== megaservice validated ===="
261+
echo "::endgroup::"
262+
263+
echo "::group::validate_frontend"
262264
validate_frontend
263-
echo "==== frontend validated ===="
265+
echo "::endgroup::"
264266

267+
echo "::group::stop_docker"
265268
stop_docker
266-
echo y | docker system prune
269+
echo "::endgroup::"
270+
271+
docker system prune -f
267272

268273
}
269274

‎ChatQnA/tests/test_compose_faqgen_on_xeon.sh

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,12 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020
cd $WORKPATH/docker_image_build
3121
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
22+
pushd GenAIComps
23+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
24+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
25+
popd && sleep 1s
3226
git clone https://github.com/vllm-project/vllm.git && cd vllm
3327
VLLM_VER="$(git describe --tags "$(git rev-list --tags --max-count=1)" )"
3428
echo "Check out vLLM tag ${VLLM_VER}"
@@ -250,21 +244,35 @@ function stop_docker() {
250244

251245
function main() {
252246

247+
echo "::group::start_docker"
253248
stop_docker
249+
echo "::endgroup::"
250+
251+
echo "::group::build_docker_images"
254252
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
255-
start_time=$(date +%s)
253+
echo "::endgroup::"
254+
255+
echo "::group::start_services"
256256
start_services
257-
end_time=$(date +%s)
258-
duration=$((end_time-start_time))
259-
echo "Mega service start duration is $duration s" && sleep 1s
257+
echo "::endgroup::"
260258

259+
echo "::group::validate_microservices"
261260
validate_microservices
261+
echo "::endgroup::"
262+
263+
echo "::group::validate_megaservice"
262264
validate_megaservice
265+
echo "::endgroup::"
266+
267+
echo "::group::validate_frontend"
263268
validate_frontend
269+
echo "::endgroup::"
264270

271+
echo "::group::stop_docker"
265272
stop_docker
266-
echo y | docker system prune
273+
echo "::endgroup::"
267274

275+
docker system prune -f
268276
}
269277

270278
main

‎ChatQnA/tests/test_compose_faqgen_tgi_on_gaudi.sh

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,12 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020
cd $WORKPATH/docker_image_build
3121
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
22+
pushd GenAIComps
23+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
24+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
25+
popd && sleep 1s
3226

3327
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3428
service_list="chatqna chatqna-ui dataprep retriever llm-faqgen nginx"
@@ -246,20 +240,35 @@ function stop_docker() {
246240

247241
function main() {
248242

243+
echo "::group::start_docker"
249244
stop_docker
245+
echo "::endgroup::"
246+
247+
echo "::group::build_docker_images"
250248
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
251-
start_time=$(date +%s)
249+
echo "::endgroup::"
250+
251+
echo "::group::start_services"
252252
start_services
253-
end_time=$(date +%s)
254-
duration=$((end_time-start_time))
255-
echo "Mega service start duration is $duration s"
253+
echo "::endgroup::"
256254

255+
echo "::group::validate_microservices"
257256
validate_microservices
257+
echo "::endgroup::"
258+
259+
echo "::group::validate_megaservice"
258260
validate_megaservice
261+
echo "::endgroup::"
262+
263+
echo "::group::validate_frontend"
259264
validate_frontend
265+
echo "::endgroup::"
260266

267+
echo "::group::stop_docker"
261268
stop_docker
262-
echo y | docker system prune
269+
echo "::endgroup::"
270+
271+
docker system prune -f
263272

264273
}
265274

‎ChatQnA/tests/test_compose_faqgen_tgi_on_xeon.sh

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,12 @@ ip_address=$(hostname -I | awk '{print $1}')
1717

1818
function build_docker_images() {
1919
opea_branch=${opea_branch:-"main"}
20-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
21-
if [[ "${opea_branch}" != "main" ]]; then
22-
cd $WORKPATH
23-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
24-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
25-
find . -type f -name "Dockerfile*" | while read -r file; do
26-
echo "Processing file: $file"
27-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
28-
done
29-
fi
3020
cd $WORKPATH/docker_image_build
3121
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
22+
pushd GenAIComps
23+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
24+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
25+
popd && sleep 1s
3226
git clone https://github.com/vllm-project/vllm.git && cd vllm
3327
VLLM_VER="$(git describe --tags "$(git rev-list --tags --max-count=1)" )"
3428
echo "Check out vLLM tag ${VLLM_VER}"
@@ -250,20 +244,35 @@ function stop_docker() {
250244

251245
function main() {
252246

247+
echo "::group::start_docker"
253248
stop_docker
249+
echo "::endgroup::"
250+
251+
echo "::group::build_docker_images"
254252
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
255-
start_time=$(date +%s)
253+
echo "::endgroup::"
254+
255+
echo "::group::start_services"
256256
start_services
257-
end_time=$(date +%s)
258-
duration=$((end_time-start_time))
259-
echo "Mega service start duration is $duration s" && sleep 1s
257+
echo "::endgroup::"
260258

259+
echo "::group::validate_microservices"
261260
validate_microservices
261+
echo "::endgroup::"
262+
263+
echo "::group::validate_megaservice"
262264
validate_megaservice
265+
echo "::endgroup::"
266+
267+
echo "::group::validate_frontend"
263268
validate_frontend
269+
echo "::endgroup::"
264270

271+
echo "::group::stop_docker"
265272
stop_docker
266-
echo y | docker system prune
273+
echo "::endgroup::"
274+
275+
docker system prune -f
267276

268277
}
269278

0 commit comments

Comments
 (0)