Skip to content

Commit d92968e

Browse files
louie-tsailouie-tsai
committed
Enable OpenTelemtry Tracing for ChatQnA on Xeon and Gaudi
Signed-off-by: Louie, Tsai <louie.tsai@intel.com> Signed-off-by: Tsai, Louie <louie.tsai@intel.com>
1 parent fdbc27a commit d92968e

File tree

6 files changed

+69
-8
lines changed

6 files changed

+69
-8
lines changed

‎ChatQnA/docker_compose/intel/cpu/xeon/compose.yaml‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ services:
3737
no_proxy: ${no_proxy}
3838
http_proxy: ${http_proxy}
3939
https_proxy: ${https_proxy}
40-
command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
40+
command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
4141
retriever:
4242
image: ${REGISTRY:-opea}/retriever:${TAG:-latest}
4343
container_name: retriever-redis-server
@@ -73,7 +73,7 @@ services:
7373
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
7474
HF_HUB_DISABLE_PROGRESS_BARS: 1
7575
HF_HUB_ENABLE_HF_TRANSFER: 0
76-
command: --model-id ${RERANK_MODEL_ID} --auto-truncate
76+
command: --model-id ${RERANK_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
7777
vllm-service:
7878
image: ${REGISTRY:-opea}/vllm:${TAG:-latest}
7979
container_name: vllm-service
@@ -95,6 +95,21 @@ services:
9595
timeout: 10s
9696
retries: 100
9797
command: --model $LLM_MODEL_ID --host 0.0.0.0 --port 80
98+
jaeger:
99+
image: jaegertracing/all-in-one:latest
100+
container_name: jaeger
101+
ports:
102+
- "16686:16686"
103+
- "4317:4317"
104+
- "4318:4318"
105+
- "9411:9411"
106+
ipc: host
107+
environment:
108+
no_proxy: ${no_proxy}
109+
http_proxy: ${http_proxy}
110+
https_proxy: ${https_proxy}
111+
COLLECTOR_ZIPKIN_HOST_PORT: 9411
112+
restart: unless-stopped
98113
chatqna-xeon-backend-server:
99114
image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
100115
container_name: chatqna-xeon-backend-server
@@ -120,6 +135,8 @@ services:
120135
- LLM_SERVER_PORT=${LLM_SERVER_PORT:-80}
121136
- LLM_MODEL=${LLM_MODEL_ID}
122137
- LOGFLAG=${LOGFLAG}
138+
- ENABLE_OPEA_TELEMETRY=true
139+
- TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT}
123140
ipc: host
124141
restart: always
125142
chatqna-xeon-ui-server:

‎ChatQnA/docker_compose/intel/cpu/xeon/compose_tgi.yaml‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ services:
3737
no_proxy: ${no_proxy}
3838
http_proxy: ${http_proxy}
3939
https_proxy: ${https_proxy}
40-
command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
40+
command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
4141
retriever:
4242
image: ${REGISTRY:-opea}/retriever:${TAG:-latest}
4343
container_name: retriever-redis-server
@@ -73,7 +73,7 @@ services:
7373
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
7474
HF_HUB_DISABLE_PROGRESS_BARS: 1
7575
HF_HUB_ENABLE_HF_TRANSFER: 0
76-
command: --model-id ${RERANK_MODEL_ID} --auto-truncate
76+
command: --model-id ${RERANK_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
7777
tgi-service:
7878
image: ghcr.io/huggingface/text-generation-inference:2.4.0-intel-cpu
7979
container_name: tgi-service
@@ -89,7 +89,22 @@ services:
8989
HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
9090
HF_HUB_DISABLE_PROGRESS_BARS: 1
9191
HF_HUB_ENABLE_HF_TRANSFER: 0
92-
command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0
92+
command: --model-id ${LLM_MODEL_ID} --cuda-graphs 0 --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
93+
jaeger:
94+
image: jaegertracing/all-in-one:latest
95+
container_name: jaeger
96+
ports:
97+
- "16686:16686"
98+
- "4317:4317"
99+
- "4318:4318"
100+
- "9411:9411"
101+
ipc: host
102+
environment:
103+
no_proxy: ${no_proxy}
104+
http_proxy: ${http_proxy}
105+
https_proxy: ${https_proxy}
106+
COLLECTOR_ZIPKIN_HOST_PORT: 9411
107+
restart: unless-stopped
93108
chatqna-xeon-backend-server:
94109
image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
95110
container_name: chatqna-xeon-backend-server
@@ -116,6 +131,8 @@ services:
116131
- LLM_SERVER_PORT=${LLM_SERVER_PORT:-80}
117132
- LLM_MODEL=${LLM_MODEL_ID}
118133
- LOGFLAG=${LOGFLAG}
134+
- ENABLE_OPEA_TELEMETRY=true
135+
- TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT}
119136
ipc: host
120137
restart: always
121138
chatqna-xeon-ui-server:

‎ChatQnA/docker_compose/intel/cpu/xeon/set_env.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,7 @@ export INDEX_NAME="rag-redis"
1414
# Set it as a non-null string, such as true, if you want to enable logging facility,
1515
# otherwise, keep it as "" to disable it.
1616
export LOGFLAG=""
17+
# Set OpenTelemetry Tracing Endpoint
18+
export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
19+
export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
20+
export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces

‎ChatQnA/docker_compose/intel/hpu/gaudi/compose.yaml‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ services:
2525
INDEX_NAME: ${INDEX_NAME}
2626
TEI_ENDPOINT: http://tei-embedding-service:80
2727
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
28+
TELEMETRY_ENDPOINT: ${TELEMETRY_ENDPOINT}
2829
tei-embedding-service:
2930
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5
3031
container_name: tei-embedding-gaudi-server
@@ -37,7 +38,7 @@ services:
3738
no_proxy: ${no_proxy}
3839
http_proxy: ${http_proxy}
3940
https_proxy: ${https_proxy}
40-
command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
41+
command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
4142
retriever:
4243
image: ${REGISTRY:-opea}/retriever:${TAG:-latest}
4344
container_name: retriever-redis-server
@@ -55,6 +56,7 @@ services:
5556
INDEX_NAME: ${INDEX_NAME}
5657
TEI_EMBEDDING_ENDPOINT: http://tei-embedding-service:80
5758
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
59+
TELEMETRY_ENDPOINT: ${TELEMETRY_ENDPOINT}
5860
restart: unless-stopped
5961
tei-reranking-service:
6062
image: ghcr.io/huggingface/tei-gaudi:1.5.0
@@ -76,7 +78,7 @@ services:
7678
HABANA_VISIBLE_DEVICES: all
7779
OMPI_MCA_btl_vader_single_copy_mechanism: none
7880
MAX_WARMUP_SEQUENCE_LENGTH: 512
79-
command: --model-id ${RERANK_MODEL_ID} --auto-truncate
81+
command: --model-id ${RERANK_MODEL_ID} --auto-truncate --otlp-endpoint $OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
8082
vllm-service:
8183
image: ${REGISTRY:-opea}/vllm-gaudi:${TAG:-latest}
8284
container_name: vllm-gaudi-server
@@ -97,12 +99,27 @@ services:
9799
test: ["CMD-SHELL", "curl -f http://$host_ip:8007/health || exit 1"]
98100
interval: 10s
99101
timeout: 10s
100-
retries: 100
102+
retries: 10 --otlp-traces-endpoint=0
101103
runtime: habana
102104
cap_add:
103105
- SYS_NICE
104106
ipc: host
105107
command: --model $LLM_MODEL_ID --tensor-parallel-size 1 --host 0.0.0.0 --port 80 --block-size 128 --max-num-seqs 256 --max-seq_len-to-capture 2048
108+
jaeger:
109+
image: jaegertracing/all-in-one:latest
110+
container_name: jaeger
111+
ports:
112+
- "16686:16686"
113+
- "4317:4317"
114+
- "4318:4318"
115+
- "9411:9411"
116+
ipc: host
117+
environment:
118+
no_proxy: ${no_proxy}
119+
http_proxy: ${http_proxy}
120+
https_proxy: ${https_proxy}
121+
COLLECTOR_ZIPKIN_HOST_PORT: 9411
122+
restart: unless-stopped
106123
chatqna-gaudi-backend-server:
107124
image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
108125
container_name: chatqna-gaudi-backend-server
@@ -128,6 +145,8 @@ services:
128145
- LLM_SERVER_PORT=${LLM_SERVER_PORT:-80}
129146
- LLM_MODEL=${LLM_MODEL_ID}
130147
- LOGFLAG=${LOGFLAG}
148+
- ENABLE_OPEA_TELEMETRY=true
149+
- TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT}
131150
ipc: host
132151
restart: always
133152
chatqna-gaudi-ui-server:

‎ChatQnA/docker_compose/intel/hpu/gaudi/compose_tgi.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ services:
146146
- LLM_SERVER_PORT=${LLM_SERVER_PORT:-80}
147147
- LLM_MODEL=${LLM_MODEL_ID}
148148
- LOGFLAG=${LOGFLAG}
149+
- ENABLE_OPEA_TELEMETRY=true
149150
- TELEMETRY_ENDPOINT=${TELEMETRY_ENDPOINT}
150151
ipc: host
151152
restart: always

‎ChatQnA/tests/test_compose_on_xeon.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ function start_services() {
4949
export INDEX_NAME="rag-redis"
5050
export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
5151
export host_ip=${ip_address}
52+
export JAEGER_IP=$(ip route get 8.8.8.8 | grep -oP 'src \K[^ ]+')
53+
export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=grpc://$JAEGER_IP:4317
54+
export TELEMETRY_ENDPOINT=http://$JAEGER_IP:4318/v1/traces
5255

5356
# Start Docker Containers
5457
docker compose -f compose.yaml up -d > ${LOG_PATH}/start_services_with_compose.log

0 commit comments

Comments
 (0)