Skip to content

Commit

Permalink
Refine docker_compose for dataprep param settings (#486)
Browse files Browse the repository at this point in the history
* add TEI_ENDPOINT for dataprep

Signed-off-by: letonghan <letong.han@intel.com>

* add TEI_ENDPOINT for dataprep

Signed-off-by: letonghan <letong.han@intel.com>

* update ui test file

Signed-off-by: letonghan <letong.han@intel.com>

* add dataprep test into script

Signed-off-by: letonghan <letong.han@intel.com>

* refine e2e test of chatqna

Signed-off-by: letonghan <letong.han@intel.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix missing dockerfile path

Signed-off-by: letonghan <letong.han@intel.com>

* check test issue

Signed-off-by: letonghan <letong.han@intel.com>

* check dataprep log

Signed-off-by: letonghan <letong.han@intel.com>

* reverse codetrans

Signed-off-by: letonghan <letong.han@intel.com>

* cd log path

Signed-off-by: letonghan <letong.han@intel.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add hf token in compose yaml

Signed-off-by: letonghan <letong.han@intel.com>

* add redis_host

Signed-off-by: letonghan <letong.han@intel.com>

* add upload link test

Signed-off-by: letonghan <letong.han@intel.com>

* re-format validate dataprep

Signed-off-by: letonghan <letong.han@intel.com>

* fix typo

Signed-off-by: letonghan <letong.han@intel.com>

* fix frontend env name

Signed-off-by: letonghan <letong.han@intel.com>

* fix ci issue

Signed-off-by: letonghan <letong.han@intel.com>

---------

Signed-off-by: letonghan <letong.han@intel.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
letonghan and pre-commit-ci[bot] authored Aug 6, 2024
1 parent abc02e1 commit 3913c7b
Show file tree
Hide file tree
Showing 10 changed files with 183 additions and 64 deletions.
3 changes: 3 additions & 0 deletions ChatQnA/docker/gaudi/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ services:
container_name: dataprep-redis-server
depends_on:
- redis-vector-db
- tei-embedding-service
ports:
- "6007:6007"
- "6008:6008"
Expand All @@ -26,6 +27,8 @@ services:
https_proxy: ${https_proxy}
REDIS_URL: ${REDIS_URL}
INDEX_NAME: ${INDEX_NAME}
TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
tei-embedding-service:
image: opea/tei-gaudi:latest
container_name: tei-embedding-gaudi-server
Expand Down
2 changes: 2 additions & 0 deletions ChatQnA/docker/gaudi/compose_guardrails.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ services:
container_name: dataprep-redis-server
depends_on:
- redis-vector-db
- tei-embedding-service
ports:
- "6007:6007"
- "6008:6008"
Expand All @@ -26,6 +27,7 @@ services:
https_proxy: ${https_proxy}
REDIS_URL: ${REDIS_URL}
INDEX_NAME: ${INDEX_NAME}
TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
tgi-guardrails-service:
image: ghcr.io/huggingface/tgi-gaudi:2.0.1
container_name: tgi-guardrails-server
Expand Down
2 changes: 2 additions & 0 deletions ChatQnA/docker/gaudi/compose_vllm.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ services:
container_name: dataprep-redis-server
depends_on:
- redis-vector-db
- tei-embedding-service
ports:
- "6007:6007"
- "6008:6008"
Expand All @@ -26,6 +27,7 @@ services:
https_proxy: ${https_proxy}
REDIS_URL: ${REDIS_URL}
INDEX_NAME: ${INDEX_NAME}
TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
tei-embedding-service:
image: opea/tei-gaudi:latest
container_name: tei-embedding-gaudi-server
Expand Down
2 changes: 2 additions & 0 deletions ChatQnA/docker/gaudi/compose_vllm_ray.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ services:
container_name: dataprep-redis-server
depends_on:
- redis-vector-db
- tei-embedding-service
ports:
- "6007:6007"
- "6008:6008"
Expand All @@ -26,6 +27,7 @@ services:
https_proxy: ${https_proxy}
REDIS_URL: ${REDIS_URL}
INDEX_NAME: ${INDEX_NAME}
TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
tei-embedding-service:
image: opea/tei-gaudi:latest
container_name: tei-embedding-gaudi-server
Expand Down
4 changes: 4 additions & 0 deletions ChatQnA/docker/gpu/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ services:
container_name: dataprep-redis-server
depends_on:
- redis-vector-db
- tei-embedding-service
ports:
- "6007:6007"
- "6008:6008"
Expand All @@ -25,7 +26,10 @@ services:
http_proxy: ${http_proxy}
https_proxy: ${https_proxy}
REDIS_URL: ${REDIS_URL}
REDIS_HOST: ${REDIS_HOST}
INDEX_NAME: ${INDEX_NAME}
TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
tei-embedding-service:
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
container_name: tei-embedding-server
Expand Down
5 changes: 5 additions & 0 deletions ChatQnA/docker/xeon/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ services:
container_name: dataprep-redis-server
depends_on:
- redis-vector-db
- tei-embedding-service
ports:
- "6007:6007"
- "6008:6008"
Expand All @@ -25,7 +26,10 @@ services:
http_proxy: ${http_proxy}
https_proxy: ${https_proxy}
REDIS_URL: ${REDIS_URL}
REDIS_HOST: ${REDIS_HOST}
INDEX_NAME: ${INDEX_NAME}
TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
tei-embedding-service:
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
container_name: tei-embedding-server
Expand Down Expand Up @@ -154,6 +158,7 @@ services:
- redis-vector-db
- tei-embedding-service
- embedding
- dataprep-redis-service
- retriever
- tei-reranking-service
- reranking
Expand Down
2 changes: 2 additions & 0 deletions ChatQnA/docker/xeon/docker_compose_qdrant.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ services:
container_name: dataprep-qdrant-server
depends_on:
- qdrant-vector-db
- tei-embedding-service
ports:
- "6000:6000"
environment:
Expand All @@ -25,6 +26,7 @@ services:
QDRANT: ${host_ip}
QDRANT_PORT: 6333
COLLECTION_NAME: ${INDEX_NAME}
TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
tei-embedding-service:
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.2
container_name: tei-embedding-server
Expand Down
2 changes: 2 additions & 0 deletions ChatQnA/docker/xeon/docker_compose_vllm.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ services:
container_name: dataprep-redis-server
depends_on:
- redis-vector-db
- tei-embedding-service
ports:
- "6007:6007"
- "6008:6008"
Expand All @@ -26,6 +27,7 @@ services:
https_proxy: ${https_proxy}
REDIS_URL: ${REDIS_URL}
INDEX_NAME: ${INDEX_NAME}
TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
tei-embedding-service:
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.2
container_name: tei-embedding-server
Expand Down
110 changes: 78 additions & 32 deletions ChatQnA/tests/test_chatqna_on_gaudi.sh
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ function start_services() {
export TEI_RERANKING_ENDPOINT="http://${ip_address}:8808"
export TGI_LLM_ENDPOINT="http://${ip_address}:8008"
export REDIS_URL="redis://${ip_address}:6379"
export REDIS_HOST=${ip_address}
export INDEX_NAME="rag-redis"
export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
export MEGA_SERVICE_HOST_IP=${ip_address}
Expand All @@ -61,6 +62,8 @@ function start_services() {
export LLM_SERVICE_HOST_IP=${ip_address}
export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/chatqna"
export DATAPREP_SERVICE_ENDPOINT="http://${ip_address}:6007/v1/dataprep"
export DATAPREP_GET_FILE_ENDPOINT="http://${ip_address}:6008/v1/dataprep/get_file"
export DATAPREP_DELETE_FILE_ENDPOINT="http://${ip_address}:6009/v1/dataprep/delete_file"

sed -i "s/backend_address/$ip_address/g" $WORKPATH/docker/ui/svelte/.env

Expand Down Expand Up @@ -93,104 +96,147 @@ function start_services() {
done
}

function validate_services() {
function validate_service() {
local URL="$1"
local EXPECTED_RESULT="$2"
local SERVICE_NAME="$3"
local DOCKER_NAME="$4"
local INPUT_DATA="$5"

local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
if [ "$HTTP_STATUS" -eq 200 ]; then
echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
if [[ $SERVICE_NAME == *"dataprep_upload_file"* ]]; then
cd $LOG_PATH
HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -F 'files=@./dataprep_file.txt' -H 'Content-Type: multipart/form-data' "$URL")
elif [[ $SERVICE_NAME == *"dataprep_upload_link"* ]]; then
HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -F 'link_list=["https://www.ces.tech/"]' "$URL")
elif [[ $SERVICE_NAME == *"dataprep_get"* ]]; then
HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -H 'Content-Type: application/json' "$URL")
elif [[ $SERVICE_NAME == *"dataprep_del"* ]]; then
HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -d '{"file_path": "all"}' -H 'Content-Type: application/json' "$URL")
else
HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL")
fi
HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')

local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log)
docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log

if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
echo "[ $SERVICE_NAME ] Content is as expected."
else
echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
exit 1
fi
else
# check response status
if [ "$HTTP_STATUS" -ne "200" ]; then
echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log
exit 1
else
echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
fi
# check response body
if [[ "$RESPONSE_BODY" != *"$EXPECTED_RESULT"* ]]; then
echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY"
exit 1
else
echo "[ $SERVICE_NAME ] Content is as expected."
fi

sleep 1s
}

function validate_microservices() {
# Check if the microservices are running correctly.

# tei for embedding service
validate_services \
validate_service \
"${ip_address}:8090/embed" \
"\[\[" \
"[[" \
"tei-embedding" \
"tei-embedding-gaudi-server" \
'{"inputs":"What is Deep Learning?"}'

# embedding microservice
validate_services \
validate_service \
"${ip_address}:6000/v1/embeddings" \
'"text":"What is Deep Learning?","embedding":\[' \
"embedding" \
'"text":"What is Deep Learning?","embedding":[' \
"embedding-microservice" \
"embedding-tei-server" \
'{"text":"What is Deep Learning?"}'

sleep 1m # retrieval can't curl as expected, try to wait for more time

# test /v1/dataprep upload file
echo "Deep learning is a subset of machine learning that utilizes neural networks with multiple layers to analyze various levels of abstract data representations. It enables computers to identify patterns and make decisions with minimal human intervention by learning from large amounts of data." > $LOG_PATH/dataprep_file.txt
validate_service \
"http://${ip_address}:6007/v1/dataprep" \
"Data preparation succeeded" \
"dataprep_upload_file" \
"dataprep-redis-server"

# test /v1/dataprep upload link
validate_service \
"http://${ip_address}:6007/v1/dataprep" \
"Data preparation succeeded" \
"dataprep_upload_link" \
"dataprep-redis-server"

# test /v1/dataprep/get_file
validate_service \
"http://${ip_address}:6008/v1/dataprep/get_file" \
'{"name":' \
"dataprep_get" \
"dataprep-redis-server"

# test /v1/dataprep/delete_file
validate_service \
"http://${ip_address}:6009/v1/dataprep/delete_file" \
'{"status":true}' \
"dataprep_del" \
"dataprep-redis-server"

# retrieval microservice
test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)")
validate_services \
validate_service \
"${ip_address}:7000/v1/retrieval" \
" " \
"retrieval" \
"retrieved_docs" \
"retrieval-microservice" \
"retriever-redis-server" \
"{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}"

# tei for rerank microservice
validate_services \
validate_service \
"${ip_address}:8808/rerank" \
'{"index":1,"score":' \
"tei-rerank" \
"tei-reranking-gaudi-server" \
'{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}'

# rerank microservice
validate_services \
validate_service \
"${ip_address}:8000/v1/reranking" \
"Deep learning is..." \
"rerank" \
"rerank-microservice" \
"reranking-tei-gaudi-server" \
'{"initial_query":"What is Deep Learning?", "retrieved_docs": [{"text":"Deep Learning is not..."}, {"text":"Deep learning is..."}]}'

# tgi for llm service
validate_services \
validate_service \
"${ip_address}:8008/generate" \
"generated_text" \
"tgi-llm" \
"tgi-gaudi-server" \
'{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}'

# llm microservice
validate_services \
validate_service \
"${ip_address}:9000/v1/chat/completions" \
"data: " \
"llm" \
"llm-microservice" \
"llm-tgi-gaudi-server" \
'{"query":"What is Deep Learning?"}'

}

function validate_megaservice() {
# Curl the Mega Service
validate_services \
validate_service \
"${ip_address}:8888/v1/chatqna" \
"billion" \
"mega-chatqna" \
"data: " \
"chatqna-megaservice" \
"chatqna-gaudi-backend-server" \
'{"messages": "What is the revenue of Nike in 2023?"}'

Expand Down Expand Up @@ -241,7 +287,7 @@ function main() {
elif [ "${mode}" == "" ]; then
validate_microservices
validate_megaservice
# validate_frontend
validate_frontend
fi

stop_docker
Expand Down
Loading

0 comments on commit 3913c7b

Please sign in to comment.