From 39135b2df392ee78e2da838f77d55adc6613f5b5 Mon Sep 17 00:00:00 2001 From: BaoHuiling Date: Thu, 8 Aug 2024 11:59:11 +0800 Subject: [PATCH 01/15] initial commit Signed-off-by: BaoHuiling --- comps/__init__.py | 4 + comps/cores/proto/docarray.py | 26 +++++- comps/reranks/video-rag-qna/docker/Dockerfile | 24 ++++++ .../reranks/video-rag-qna/local_reranking.py | 82 +++++++++++++++++++ comps/reranks/video-rag-qna/requirements.txt | 11 +++ 5 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 comps/reranks/video-rag-qna/docker/Dockerfile create mode 100644 comps/reranks/video-rag-qna/local_reranking.py create mode 100644 comps/reranks/video-rag-qna/requirements.txt diff --git a/comps/__init__.py b/comps/__init__.py index cb7ed7a28..fbe34f7fa 100644 --- a/comps/__init__.py +++ b/comps/__init__.py @@ -12,12 +12,16 @@ GeneratedDoc, LLMParamsDoc, SearchedDoc, + SearchedMultimodalDoc, RerankedDoc, TextDoc, + ImageDoc, + TextImageDoc, RAGASParams, RAGASScores, GraphDoc, LVMDoc, + LVMVideoDoc, ) # Constants diff --git a/comps/cores/proto/docarray.py b/comps/cores/proto/docarray.py index 9760d7d3e..d4ebe90bf 100644 --- a/comps/cores/proto/docarray.py +++ b/comps/cores/proto/docarray.py @@ -1,11 +1,15 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +<<<<<<< HEAD from typing import Dict, List, Optional, Union +======= +from typing import Optional, Tuple, Union +>>>>>>> fddc313 (initial commit) import numpy as np from docarray import BaseDoc, DocList -from docarray.documents import AudioDoc +from docarray.documents import AudioDoc, VideoDoc from docarray.typing import AudioUrl from pydantic import Field, conint, conlist, field_validator @@ -19,6 +23,11 @@ class TopologyInfo: class TextDoc(BaseDoc, TopologyInfo): text: str +class ImageDoc(BaseDoc): + image_path: str + +class TextImageDoc(BaseDoc): + doc: Tuple[Union[TextDoc, ImageDoc]] class Base64ByteStrDoc(BaseDoc): byte_str: str @@ -66,6 +75,14 @@ class SearchedDoc(BaseDoc): class Config: json_encoders = {np.ndarray: lambda x: x.tolist()} +class SearchedMultimodalDoc(BaseDoc): + retrieved_docs: DocList[TextImageDoc] + initial_query: str + top_n: int = 1 + metadata: Optional[DocList[dict]] = [{}] + + class Config: + json_encoders = {np.ndarray: lambda x: x.tolist()} class GeneratedDoc(BaseDoc): text: str @@ -170,3 +187,10 @@ class LVMDoc(BaseDoc): temperature: float = 0.01 repetition_penalty: float = 1.03 streaming: bool = False + +class LVMVideoDoc(BaseDoc): + video_url: str + chunck_start: float + chunck_duration: float + prompt: str + max_new_tokens: conint(ge=0, le=1024) = 512 diff --git a/comps/reranks/video-rag-qna/docker/Dockerfile b/comps/reranks/video-rag-qna/docker/Dockerfile new file mode 100644 index 000000000..617f47b6a --- /dev/null +++ b/comps/reranks/video-rag-qna/docker/Dockerfile @@ -0,0 +1,24 @@ + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FROM python:3.11-slim + +ENV LANG=C.UTF-8 + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +USER user + +COPY comps /home/user/comps + +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r /home/user/comps/reranks/video-rag-qna/requirements.txt + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +WORKDIR /home/user/comps/reranks/video-rag-qna + +ENTRYPOINT ["python", "local_reranking.py"] \ No newline at end of file diff --git a/comps/reranks/video-rag-qna/local_reranking.py b/comps/reranks/video-rag-qna/local_reranking.py new file mode 100644 index 000000000..9983609be --- /dev/null +++ b/comps/reranks/video-rag-qna/local_reranking.py @@ -0,0 +1,82 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import logging +import os +import time + +from langsmith import traceable + +from comps import ( + SearchedMultimodalDoc, + LVMVideoDoc, + ServiceType, + opea_microservices, + register_microservice, + register_statistics, + statistics_dict, +) + +chunck_duration = os.getenv("CHUNCK_DURATION", 10) + +logging.basicConfig( + level=logging.INFO, + format="%(levelname)s: [%(asctime)s] %(message)s", + datefmt="%d/%m/%Y %I:%M:%S" + ) + +def get_top_doc(top_n, videos) -> list: + hit_score = {} + if videos == None: + return None + for video_name in videos: + try: + if video_name not in hit_score.keys(): + hit_score[video_name] = 0 + hit_score[video_name] += 1 + except KeyError as r: + logging.info(f"no video name {r}") + + x = dict(sorted(hit_score.items(), key=lambda item: -item[1])) # sorted dict of video name and score + top_n_names = list(x.keys())[:top_n] + logging.info(f"top docs = {x}") + logging.info(f"top n docs names = {top_n_names}") + + return top_n_names + +def find_timestamp_from_video(metadata_list, video): + for metadata in metadata_list: + if metadata['video'] == video: + return metadata['timestamp'] + return None + +@register_microservice( + name="opea_service@reranking_visual_rag", + service_type=ServiceType.RERANK, + endpoint="/v1/reranking", + host="0.0.0.0", + port=8000, + input_datatype=SearchedMultimodalDoc, + output_datatype=LVMVideoDoc, +) +@traceable(run_type="rerank") +@register_statistics(names=["opea_service@reranking_visual_rag"]) +def reranking(input: SearchedMultimodalDoc) -> LVMVideoDoc: + start = time.time() + + # get top video name from metadata + video_names = [meta["video"] for meta in input.metadata] + top_video_names = get_top_doc(input.top_n, video_names) + + # only use the first top video + timestamp = find_timestamp_from_video(input.metadata, top_video_names[0]) + + result = LVMVideoDoc(video_url="TODO", prompt=input.initial_query, chunck_start=timestamp, chunck_duration=float(chunck_duration), max_new_tokens=512) + statistics_dict["opea_service@reranking_visual_rag"].append_latency(time.time() - start, None) + + return result + + +if __name__ == "__main__": + opea_microservices["opea_service@reranking_visual_rag"].start() + diff --git a/comps/reranks/video-rag-qna/requirements.txt b/comps/reranks/video-rag-qna/requirements.txt new file mode 100644 index 000000000..01c09f72d --- /dev/null +++ b/comps/reranks/video-rag-qna/requirements.txt @@ -0,0 +1,11 @@ +datasets +docarray +fastapi +uvicorn +opentelemetry-api +opentelemetry-exporter-otlp +opentelemetry-sdk +prometheus-fastapi-instrumentator +pydub +shortuuid +langsmith \ No newline at end of file From c3e415443c14e54a288c0db3fe9b2afdc20e18d2 Mon Sep 17 00:00:00 2001 From: BaoHuiling Date: Thu, 8 Aug 2024 13:28:22 +0800 Subject: [PATCH 02/15] save Signed-off-by: BaoHuiling --- comps/cores/proto/docarray.py | 4 ++-- .../docker/docker_compose_reranking.yaml | 22 +++++++++++++++++++ .../reranks/video-rag-qna/local_reranking.py | 7 +++--- 3 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml diff --git a/comps/cores/proto/docarray.py b/comps/cores/proto/docarray.py index d4ebe90bf..847539c64 100644 --- a/comps/cores/proto/docarray.py +++ b/comps/cores/proto/docarray.py @@ -190,7 +190,7 @@ class LVMDoc(BaseDoc): class LVMVideoDoc(BaseDoc): video_url: str - chunck_start: float - chunck_duration: float + chunk_start: float + chunk_duration: float prompt: str max_new_tokens: conint(ge=0, le=1024) = 512 diff --git a/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml b/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml new file mode 100644 index 000000000..f5fb93f03 --- /dev/null +++ b/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml @@ -0,0 +1,22 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +version: "3.8" + +services: + reranking: + image: opea/reranking-videoragqna:latest + container_name: reranking-videoragqna-server + ports: + - "8000:8000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + CHUNK_DURATION: ${CHUNK_DURATION} + restart: unless-stopped + +networks: + default: + driver: bridge diff --git a/comps/reranks/video-rag-qna/local_reranking.py b/comps/reranks/video-rag-qna/local_reranking.py index 9983609be..7c39a6a5e 100644 --- a/comps/reranks/video-rag-qna/local_reranking.py +++ b/comps/reranks/video-rag-qna/local_reranking.py @@ -17,7 +17,9 @@ statistics_dict, ) -chunck_duration = os.getenv("CHUNCK_DURATION", 10) +chunk_duration = os.getenv("CHUNK_DURATION", "10") +chunk_duration = chunk_duration.strip() or "10" +chunk_duration = float(chunk_duration) if chunk_duration.isdigit() else 10.0 logging.basicConfig( level=logging.INFO, @@ -71,7 +73,7 @@ def reranking(input: SearchedMultimodalDoc) -> LVMVideoDoc: # only use the first top video timestamp = find_timestamp_from_video(input.metadata, top_video_names[0]) - result = LVMVideoDoc(video_url="TODO", prompt=input.initial_query, chunck_start=timestamp, chunck_duration=float(chunck_duration), max_new_tokens=512) + result = LVMVideoDoc(video_url="TODO", prompt=input.initial_query, chunk_start=timestamp, chunk_duration=float(chunk_duration), max_new_tokens=512) statistics_dict["opea_service@reranking_visual_rag"].append_latency(time.time() - start, None) return result @@ -79,4 +81,3 @@ def reranking(input: SearchedMultimodalDoc) -> LVMVideoDoc: if __name__ == "__main__": opea_microservices["opea_service@reranking_visual_rag"].start() - From 876ce4d9e1c7959be60d59e7a34c792126209ec9 Mon Sep 17 00:00:00 2001 From: BaoHuiling Date: Thu, 8 Aug 2024 15:31:35 +0800 Subject: [PATCH 03/15] add readme, test script, fix bug Signed-off-by: BaoHuiling --- comps/cores/proto/docarray.py | 10 +-- comps/reranks/video-rag-qna/README.md | 59 ++++++++++++++++ .../docker/docker_compose_reranking.yaml | 2 - .../reranks/video-rag-qna/local_reranking.py | 2 +- tests/test_reranks_video-rag-qna.sh | 69 +++++++++++++++++++ 5 files changed, 132 insertions(+), 10 deletions(-) create mode 100644 comps/reranks/video-rag-qna/README.md create mode 100755 tests/test_reranks_video-rag-qna.sh diff --git a/comps/cores/proto/docarray.py b/comps/cores/proto/docarray.py index 847539c64..6fd31b909 100644 --- a/comps/cores/proto/docarray.py +++ b/comps/cores/proto/docarray.py @@ -1,11 +1,7 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -<<<<<<< HEAD -from typing import Dict, List, Optional, Union -======= -from typing import Optional, Tuple, Union ->>>>>>> fddc313 (initial commit) +from typing import Dict, List, Optional, Union, Tuple import numpy as np from docarray import BaseDoc, DocList @@ -76,10 +72,10 @@ class Config: json_encoders = {np.ndarray: lambda x: x.tolist()} class SearchedMultimodalDoc(BaseDoc): - retrieved_docs: DocList[TextImageDoc] + retrieved_docs: List[TextImageDoc] initial_query: str top_n: int = 1 - metadata: Optional[DocList[dict]] = [{}] + metadata: Optional[List[Dict]] = None class Config: json_encoders = {np.ndarray: lambda x: x.tolist()} diff --git a/comps/reranks/video-rag-qna/README.md b/comps/reranks/video-rag-qna/README.md new file mode 100644 index 000000000..84bde8a21 --- /dev/null +++ b/comps/reranks/video-rag-qna/README.md @@ -0,0 +1,59 @@ +# Rerank Microservice + +This is a Docker-based microservice that do result rerank for VideoRAGQnA use case. Local rerank is used rather than rerank model. + +# 🚀1. Start Microservice with Docker + +## 1.1 Build Images + +```bash +cd GenAIComps +docker build --no-cache -t opea/reranking-videoragqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/video-rag-qna/docker/Dockerfile . +``` + +## 1.2 Start Rerank Service + +```bash +docker compose -f comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml up -d +# wait until ready +until docker logs reranking-videoragqna-server 2>&1 | grep -q "Uvicorn running on"; do + sleep 2 +done +``` + +Available configuration by environment variable: +- CHUNK_DURATION: target chunk duration, should be aligned with VideoRAGQnA dataprep. Default 10s. + +# ✅ 2. Test + +``` bash +export ip_address=$(hostname -I | awk '{print $1}') +curl -X 'POST' \ +"http://${ip_address}:8000/v1/reranking" \ +-H 'accept: application/json' \ +-H 'Content-Type: application/json' \ +-d '{ + "retrieved_docs": [{"doc": [{"text": "this is the retrieved text"}]}], + "initial_query": "this is the query", + "top_n": 1, + "metadata": [ + {"other_key": "value", "video":"top_video_name", "timestamp":"20"}, + {"other_key": "value", "video":"second_video_name", "timestamp":"40"}, + {"other_key": "value", "video":"top_video_name", "timestamp":"20"} + ] +}' +``` + +The result should be: + +```bash +{"id":"c7a81fbc19636801520db696ff62c2d0","video_url":"TODO","chunk_start":20.0,"chunk_duration":10.0,"prompt":"this is the query","max_new_tokens":512} +``` + +# ♻️ 3. Clean + +```bash +# remove the container +cid=$(docker ps -aq --filter "name=reranking-videoragqna-server") +if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +``` diff --git a/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml b/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml index f5fb93f03..816152e87 100644 --- a/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml +++ b/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml @@ -1,8 +1,6 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -version: "3.8" - services: reranking: image: opea/reranking-videoragqna:latest diff --git a/comps/reranks/video-rag-qna/local_reranking.py b/comps/reranks/video-rag-qna/local_reranking.py index 7c39a6a5e..dcf5e9c47 100644 --- a/comps/reranks/video-rag-qna/local_reranking.py +++ b/comps/reranks/video-rag-qna/local_reranking.py @@ -61,7 +61,7 @@ def find_timestamp_from_video(metadata_list, video): input_datatype=SearchedMultimodalDoc, output_datatype=LVMVideoDoc, ) -@traceable(run_type="rerank") +@traceable(run_type="llm") @register_statistics(names=["opea_service@reranking_visual_rag"]) def reranking(input: SearchedMultimodalDoc) -> LVMVideoDoc: start = time.time() diff --git a/tests/test_reranks_video-rag-qna.sh b/tests/test_reranks_video-rag-qna.sh new file mode 100755 index 000000000..f1bea359e --- /dev/null +++ b/tests/test_reranks_video-rag-qna.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe + +WORKPATH=$(dirname "$PWD") +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + cd $WORKPATH + docker build --no-cache -t opea/reranking-videoragqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/video-rag-qna/docker/Dockerfile . +} + +function start_service() { + docker compose -f comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml up -d + + until docker logs reranking-videoragqna-server 2>&1 | grep -q "Uvicorn running on"; do + sleep 2 + done +} + +function validate_microservice() { + result=$(\ + http_proxy="" \ + curl -X 'POST' \ + "http://${ip_address}:8000/v1/reranking" \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "retrieved_docs": [ + {"doc": [{"text": "this is the retrieved text"}]} + ], + "initial_query": "this is the query", + "top_n": 1, + "metadata": [ + {"other_key": "value", "video":"top_video_name", "timestamp":"20"}, + {"other_key": "value", "video":"second_video_name", "timestamp":"40"}, + {"other_key": "value", "video":"top_video_name", "timestamp":"20"} + ] + }') + if [[ $result == *"this is the query"* ]]; then + echo "Result correct." + else + echo "Result wrong." + exit 1 + fi +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=reranking-videoragqna-server") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main From bcec3916c60959438b756f4c50c036ddabd158a5 Mon Sep 17 00:00:00 2001 From: BaoHuiling Date: Sat, 10 Aug 2024 13:27:26 +0800 Subject: [PATCH 04/15] update video URL Signed-off-by: BaoHuiling --- comps/reranks/video-rag-qna/README.md | 3 ++- .../docker/docker_compose_reranking.yaml | 1 + .../reranks/video-rag-qna/local_reranking.py | 24 ++++++++++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/comps/reranks/video-rag-qna/README.md b/comps/reranks/video-rag-qna/README.md index 84bde8a21..352a29be8 100644 --- a/comps/reranks/video-rag-qna/README.md +++ b/comps/reranks/video-rag-qna/README.md @@ -14,6 +14,7 @@ docker build --no-cache -t opea/reranking-videoragqna:latest --build-arg https_p ## 1.2 Start Rerank Service ```bash +export FILE_SERVER_URL="http://fake-file-server" docker compose -f comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml up -d # wait until ready until docker logs reranking-videoragqna-server 2>&1 | grep -q "Uvicorn running on"; do @@ -47,7 +48,7 @@ curl -X 'POST' \ The result should be: ```bash -{"id":"c7a81fbc19636801520db696ff62c2d0","video_url":"TODO","chunk_start":20.0,"chunk_duration":10.0,"prompt":"this is the query","max_new_tokens":512} +{"id":"random number","video_url":"http://fake-file-server/top_video_name","chunk_start":20.0,"chunk_duration":10.0,"prompt":"this is the query","max_new_tokens":512} ``` # ♻️ 3. Clean diff --git a/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml b/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml index 816152e87..1a2ce9218 100644 --- a/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml +++ b/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml @@ -13,6 +13,7 @@ services: http_proxy: ${http_proxy} https_proxy: ${https_proxy} CHUNK_DURATION: ${CHUNK_DURATION} + FILE_SERVER_URL: ${FILE_SERVER_URL} restart: unless-stopped networks: diff --git a/comps/reranks/video-rag-qna/local_reranking.py b/comps/reranks/video-rag-qna/local_reranking.py index dcf5e9c47..ee1fc9f10 100644 --- a/comps/reranks/video-rag-qna/local_reranking.py +++ b/comps/reranks/video-rag-qna/local_reranking.py @@ -17,10 +17,11 @@ statistics_dict, ) -chunk_duration = os.getenv("CHUNK_DURATION", "10") -chunk_duration = chunk_duration.strip() or "10" +chunk_duration = os.getenv("CHUNK_DURATION", "10") or "10" chunk_duration = float(chunk_duration) if chunk_duration.isdigit() else 10.0 +file_server_url = os.getenv("FILE_SERVER_URL") or "http://0.0.0.0:6005" + logging.basicConfig( level=logging.INFO, format="%(levelname)s: [%(asctime)s] %(message)s", @@ -29,7 +30,7 @@ def get_top_doc(top_n, videos) -> list: hit_score = {} - if videos == None: + if videos is None: return None for video_name in videos: try: @@ -47,10 +48,14 @@ def get_top_doc(top_n, videos) -> list: return top_n_names def find_timestamp_from_video(metadata_list, video): - for metadata in metadata_list: - if metadata['video'] == video: - return metadata['timestamp'] - return None + return next( + ( + metadata['timestamp'] + for metadata in metadata_list + if metadata['video'] == video + ), + None, + ) @register_microservice( name="opea_service@reranking_visual_rag", @@ -72,8 +77,9 @@ def reranking(input: SearchedMultimodalDoc) -> LVMVideoDoc: # only use the first top video timestamp = find_timestamp_from_video(input.metadata, top_video_names[0]) - - result = LVMVideoDoc(video_url="TODO", prompt=input.initial_query, chunk_start=timestamp, chunk_duration=float(chunk_duration), max_new_tokens=512) + video_url = f"{file_server_url.rstrip('/')}/{top_video_names[0]}" + + result = LVMVideoDoc(video_url=video_url, prompt=input.initial_query, chunk_start=timestamp, chunk_duration=float(chunk_duration), max_new_tokens=512) statistics_dict["opea_service@reranking_visual_rag"].append_latency(time.time() - start, None) return result From 4867802f3caf56a4f4ce39e7fc7703dafd3bca93 Mon Sep 17 00:00:00 2001 From: BaoHuiling Date: Sat, 10 Aug 2024 13:30:02 +0800 Subject: [PATCH 05/15] use default Signed-off-by: BaoHuiling --- comps/reranks/video-rag-qna/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/comps/reranks/video-rag-qna/README.md b/comps/reranks/video-rag-qna/README.md index 352a29be8..2b458404a 100644 --- a/comps/reranks/video-rag-qna/README.md +++ b/comps/reranks/video-rag-qna/README.md @@ -14,7 +14,6 @@ docker build --no-cache -t opea/reranking-videoragqna:latest --build-arg https_p ## 1.2 Start Rerank Service ```bash -export FILE_SERVER_URL="http://fake-file-server" docker compose -f comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml up -d # wait until ready until docker logs reranking-videoragqna-server 2>&1 | grep -q "Uvicorn running on"; do @@ -48,7 +47,7 @@ curl -X 'POST' \ The result should be: ```bash -{"id":"random number","video_url":"http://fake-file-server/top_video_name","chunk_start":20.0,"chunk_duration":10.0,"prompt":"this is the query","max_new_tokens":512} +{"id":"random number","video_url":"http://0.0.0.0:6005/top_video_name","chunk_start":20.0,"chunk_duration":10.0,"prompt":"this is the query","max_new_tokens":512} ``` # ♻️ 3. Clean From 5a68480fc1cf197cf557ea159dde9af9075cd427 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 11:15:53 +0000 Subject: [PATCH 06/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- comps/cores/proto/docarray.py | 10 ++++-- comps/reranks/video-rag-qna/README.md | 3 +- .../reranks/video-rag-qna/local_reranking.py | 35 ++++++++++--------- comps/reranks/video-rag-qna/requirements.txt | 4 +-- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/comps/cores/proto/docarray.py b/comps/cores/proto/docarray.py index 6fd31b909..af1b99d5b 100644 --- a/comps/cores/proto/docarray.py +++ b/comps/cores/proto/docarray.py @@ -1,7 +1,7 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, List, Optional, Union, Tuple +from typing import Dict, List, Optional, Tuple, Union import numpy as np from docarray import BaseDoc, DocList @@ -19,12 +19,15 @@ class TopologyInfo: class TextDoc(BaseDoc, TopologyInfo): text: str + class ImageDoc(BaseDoc): image_path: str - + + class TextImageDoc(BaseDoc): doc: Tuple[Union[TextDoc, ImageDoc]] + class Base64ByteStrDoc(BaseDoc): byte_str: str @@ -71,6 +74,7 @@ class SearchedDoc(BaseDoc): class Config: json_encoders = {np.ndarray: lambda x: x.tolist()} + class SearchedMultimodalDoc(BaseDoc): retrieved_docs: List[TextImageDoc] initial_query: str @@ -80,6 +84,7 @@ class SearchedMultimodalDoc(BaseDoc): class Config: json_encoders = {np.ndarray: lambda x: x.tolist()} + class GeneratedDoc(BaseDoc): text: str prompt: str @@ -184,6 +189,7 @@ class LVMDoc(BaseDoc): repetition_penalty: float = 1.03 streaming: bool = False + class LVMVideoDoc(BaseDoc): video_url: str chunk_start: float diff --git a/comps/reranks/video-rag-qna/README.md b/comps/reranks/video-rag-qna/README.md index 2b458404a..fc97faa84 100644 --- a/comps/reranks/video-rag-qna/README.md +++ b/comps/reranks/video-rag-qna/README.md @@ -22,11 +22,12 @@ done ``` Available configuration by environment variable: + - CHUNK_DURATION: target chunk duration, should be aligned with VideoRAGQnA dataprep. Default 10s. # ✅ 2. Test -``` bash +```bash export ip_address=$(hostname -I | awk '{print $1}') curl -X 'POST' \ "http://${ip_address}:8000/v1/reranking" \ diff --git a/comps/reranks/video-rag-qna/local_reranking.py b/comps/reranks/video-rag-qna/local_reranking.py index ee1fc9f10..5d7196f8d 100644 --- a/comps/reranks/video-rag-qna/local_reranking.py +++ b/comps/reranks/video-rag-qna/local_reranking.py @@ -8,8 +8,8 @@ from langsmith import traceable from comps import ( - SearchedMultimodalDoc, LVMVideoDoc, + SearchedMultimodalDoc, ServiceType, opea_microservices, register_microservice, @@ -23,10 +23,9 @@ file_server_url = os.getenv("FILE_SERVER_URL") or "http://0.0.0.0:6005" logging.basicConfig( - level=logging.INFO, - format="%(levelname)s: [%(asctime)s] %(message)s", - datefmt="%d/%m/%Y %I:%M:%S" - ) + level=logging.INFO, format="%(levelname)s: [%(asctime)s] %(message)s", datefmt="%d/%m/%Y %I:%M:%S" +) + def get_top_doc(top_n, videos) -> list: hit_score = {} @@ -40,23 +39,21 @@ def get_top_doc(top_n, videos) -> list: except KeyError as r: logging.info(f"no video name {r}") - x = dict(sorted(hit_score.items(), key=lambda item: -item[1])) # sorted dict of video name and score + x = dict(sorted(hit_score.items(), key=lambda item: -item[1])) # sorted dict of video name and score top_n_names = list(x.keys())[:top_n] logging.info(f"top docs = {x}") logging.info(f"top n docs names = {top_n_names}") - + return top_n_names + def find_timestamp_from_video(metadata_list, video): return next( - ( - metadata['timestamp'] - for metadata in metadata_list - if metadata['video'] == video - ), + (metadata["timestamp"] for metadata in metadata_list if metadata["video"] == video), None, ) + @register_microservice( name="opea_service@reranking_visual_rag", service_type=ServiceType.RERANK, @@ -70,7 +67,7 @@ def find_timestamp_from_video(metadata_list, video): @register_statistics(names=["opea_service@reranking_visual_rag"]) def reranking(input: SearchedMultimodalDoc) -> LVMVideoDoc: start = time.time() - + # get top video name from metadata video_names = [meta["video"] for meta in input.metadata] top_video_names = get_top_doc(input.top_n, video_names) @@ -78,10 +75,16 @@ def reranking(input: SearchedMultimodalDoc) -> LVMVideoDoc: # only use the first top video timestamp = find_timestamp_from_video(input.metadata, top_video_names[0]) video_url = f"{file_server_url.rstrip('/')}/{top_video_names[0]}" - - result = LVMVideoDoc(video_url=video_url, prompt=input.initial_query, chunk_start=timestamp, chunk_duration=float(chunk_duration), max_new_tokens=512) + + result = LVMVideoDoc( + video_url=video_url, + prompt=input.initial_query, + chunk_start=timestamp, + chunk_duration=float(chunk_duration), + max_new_tokens=512, + ) statistics_dict["opea_service@reranking_visual_rag"].append_latency(time.time() - start, None) - + return result diff --git a/comps/reranks/video-rag-qna/requirements.txt b/comps/reranks/video-rag-qna/requirements.txt index 01c09f72d..abc233971 100644 --- a/comps/reranks/video-rag-qna/requirements.txt +++ b/comps/reranks/video-rag-qna/requirements.txt @@ -1,11 +1,11 @@ datasets docarray fastapi -uvicorn +langsmith opentelemetry-api opentelemetry-exporter-otlp opentelemetry-sdk prometheus-fastapi-instrumentator pydub shortuuid -langsmith \ No newline at end of file +uvicorn From b92f91ca0d5fd1b44970cefb236e5582701ce9a9 Mon Sep 17 00:00:00 2001 From: BaoHuiling Date: Fri, 16 Aug 2024 11:32:05 +0800 Subject: [PATCH 07/15] update core dependency Signed-off-by: BaoHuiling --- comps/reranks/video-rag-qna/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/comps/reranks/video-rag-qna/requirements.txt b/comps/reranks/video-rag-qna/requirements.txt index abc233971..49890dac1 100644 --- a/comps/reranks/video-rag-qna/requirements.txt +++ b/comps/reranks/video-rag-qna/requirements.txt @@ -9,3 +9,4 @@ prometheus-fastapi-instrumentator pydub shortuuid uvicorn +Pillow \ No newline at end of file From b1f5dd7719730e8a02ef19e08b9bd375b5923d1e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 03:19:32 +0000 Subject: [PATCH 08/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- comps/reranks/video-rag-qna/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comps/reranks/video-rag-qna/requirements.txt b/comps/reranks/video-rag-qna/requirements.txt index 49890dac1..7f393449b 100644 --- a/comps/reranks/video-rag-qna/requirements.txt +++ b/comps/reranks/video-rag-qna/requirements.txt @@ -5,8 +5,8 @@ langsmith opentelemetry-api opentelemetry-exporter-otlp opentelemetry-sdk +Pillow prometheus-fastapi-instrumentator pydub shortuuid uvicorn -Pillow \ No newline at end of file From 08c141d0add8e6a2cf5777d0526c85539e2edd70 Mon Sep 17 00:00:00 2001 From: BaoHuiling Date: Mon, 26 Aug 2024 17:20:10 +0800 Subject: [PATCH 09/15] use p 5000 Signed-off-by: BaoHuiling --- .../docker/docker_compose_reranking.yaml | 2 +- comps/reranks/video-rag-qna/local_reranking.py | 4 ++-- tests/test_reranks_video-rag-qna.sh | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml b/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml index 1a2ce9218..d819f331a 100644 --- a/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml +++ b/comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml @@ -13,7 +13,7 @@ services: http_proxy: ${http_proxy} https_proxy: ${https_proxy} CHUNK_DURATION: ${CHUNK_DURATION} - FILE_SERVER_URL: ${FILE_SERVER_URL} + FILE_SERVER_ENDPOINT: ${FILE_SERVER_ENDPOINT} restart: unless-stopped networks: diff --git a/comps/reranks/video-rag-qna/local_reranking.py b/comps/reranks/video-rag-qna/local_reranking.py index 5d7196f8d..9ad973c63 100644 --- a/comps/reranks/video-rag-qna/local_reranking.py +++ b/comps/reranks/video-rag-qna/local_reranking.py @@ -20,7 +20,7 @@ chunk_duration = os.getenv("CHUNK_DURATION", "10") or "10" chunk_duration = float(chunk_duration) if chunk_duration.isdigit() else 10.0 -file_server_url = os.getenv("FILE_SERVER_URL") or "http://0.0.0.0:6005" +file_server_endpoint = os.getenv("FILE_SERVER_ENDPOINT") or "http://0.0.0.0:6005" logging.basicConfig( level=logging.INFO, format="%(levelname)s: [%(asctime)s] %(message)s", datefmt="%d/%m/%Y %I:%M:%S" @@ -74,7 +74,7 @@ def reranking(input: SearchedMultimodalDoc) -> LVMVideoDoc: # only use the first top video timestamp = find_timestamp_from_video(input.metadata, top_video_names[0]) - video_url = f"{file_server_url.rstrip('/')}/{top_video_names[0]}" + video_url = f"{file_server_endpoint.rstrip('/')}/{top_video_names[0]}" result = LVMVideoDoc( video_url=video_url, diff --git a/tests/test_reranks_video-rag-qna.sh b/tests/test_reranks_video-rag-qna.sh index f1bea359e..ff0a6424f 100755 --- a/tests/test_reranks_video-rag-qna.sh +++ b/tests/test_reranks_video-rag-qna.sh @@ -13,7 +13,16 @@ function build_docker_images() { } function start_service() { - docker compose -f comps/reranks/video-rag-qna/docker/docker_compose_reranking.yaml up -d + docker run -d --name "test-comps-reranking-videoragqna-server" \ + -p 5032:8000 \ + --ipc=host \ + -e no_proxy=${no_proxy} \ + -e http_proxy=${http_proxy} \ + -e https_proxy=${https_proxy} \ + -e CHUNK_DURATION=${CHUNK_DURATION} \ + -e FILE_SERVER_ENDPOINT=${FILE_SERVER_ENDPOINT} \ + opea/reranking-videoragqna:latest + until docker logs reranking-videoragqna-server 2>&1 | grep -q "Uvicorn running on"; do sleep 2 @@ -24,7 +33,7 @@ function validate_microservice() { result=$(\ http_proxy="" \ curl -X 'POST' \ - "http://${ip_address}:8000/v1/reranking" \ + "http://${ip_address}:5032/v1/reranking" \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ -d '{ @@ -48,7 +57,7 @@ function validate_microservice() { } function stop_docker() { - cid=$(docker ps -aq --filter "name=reranking-videoragqna-server") + cid=$(docker ps -aq --filter "name=test-comps-reranking*") if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi } From cbbf088acb60d782e7f212134b80227375426e0d Mon Sep 17 00:00:00 2001 From: BaoHuiling Date: Wed, 28 Aug 2024 16:45:57 +0800 Subject: [PATCH 10/15] use 5037 Signed-off-by: BaoHuiling --- tests/test_reranks_video-rag-qna.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_reranks_video-rag-qna.sh b/tests/test_reranks_video-rag-qna.sh index ff0a6424f..e5d00984b 100755 --- a/tests/test_reranks_video-rag-qna.sh +++ b/tests/test_reranks_video-rag-qna.sh @@ -14,7 +14,7 @@ function build_docker_images() { function start_service() { docker run -d --name "test-comps-reranking-videoragqna-server" \ - -p 5032:8000 \ + -p 5037:8000 \ --ipc=host \ -e no_proxy=${no_proxy} \ -e http_proxy=${http_proxy} \ @@ -33,7 +33,7 @@ function validate_microservice() { result=$(\ http_proxy="" \ curl -X 'POST' \ - "http://${ip_address}:5032/v1/reranking" \ + "http://${ip_address}:5037/v1/reranking" \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ -d '{ From 994e9dc4f39731150371874bb6d6f4c2052e5555 Mon Sep 17 00:00:00 2001 From: BaoHuiling Date: Wed, 28 Aug 2024 17:07:51 +0800 Subject: [PATCH 11/15] update ctnr name Signed-off-by: BaoHuiling --- tests/test_reranks_video-rag-qna.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_reranks_video-rag-qna.sh b/tests/test_reranks_video-rag-qna.sh index e5d00984b..cf4d0c5c8 100755 --- a/tests/test_reranks_video-rag-qna.sh +++ b/tests/test_reranks_video-rag-qna.sh @@ -24,7 +24,7 @@ function start_service() { opea/reranking-videoragqna:latest - until docker logs reranking-videoragqna-server 2>&1 | grep -q "Uvicorn running on"; do + until docker logs test-comps-reranking-videoragqna-server 2>&1 | grep -q "Uvicorn running on"; do sleep 2 done } From 0ac301e6caf67ee53722b5bb8a9335ed3f13a20b Mon Sep 17 00:00:00 2001 From: BaoHuiling Date: Thu, 29 Aug 2024 09:39:19 +0800 Subject: [PATCH 12/15] remove langsmith Signed-off-by: BaoHuiling --- comps/reranks/video-rag-qna/local_reranking.py | 4 +--- comps/reranks/video-rag-qna/requirements.txt | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/comps/reranks/video-rag-qna/local_reranking.py b/comps/reranks/video-rag-qna/local_reranking.py index 9ad973c63..5743ea721 100644 --- a/comps/reranks/video-rag-qna/local_reranking.py +++ b/comps/reranks/video-rag-qna/local_reranking.py @@ -5,8 +5,6 @@ import os import time -from langsmith import traceable - from comps import ( LVMVideoDoc, SearchedMultimodalDoc, @@ -63,7 +61,7 @@ def find_timestamp_from_video(metadata_list, video): input_datatype=SearchedMultimodalDoc, output_datatype=LVMVideoDoc, ) -@traceable(run_type="llm") + @register_statistics(names=["opea_service@reranking_visual_rag"]) def reranking(input: SearchedMultimodalDoc) -> LVMVideoDoc: start = time.time() diff --git a/comps/reranks/video-rag-qna/requirements.txt b/comps/reranks/video-rag-qna/requirements.txt index 7f393449b..c7cc250eb 100644 --- a/comps/reranks/video-rag-qna/requirements.txt +++ b/comps/reranks/video-rag-qna/requirements.txt @@ -1,7 +1,6 @@ datasets docarray fastapi -langsmith opentelemetry-api opentelemetry-exporter-otlp opentelemetry-sdk From 5149271184a818129b6db680ebac5cc5157a1bcf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 01:39:43 +0000 Subject: [PATCH 13/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- comps/reranks/video-rag-qna/local_reranking.py | 1 - 1 file changed, 1 deletion(-) diff --git a/comps/reranks/video-rag-qna/local_reranking.py b/comps/reranks/video-rag-qna/local_reranking.py index 5743ea721..3a3043ca8 100644 --- a/comps/reranks/video-rag-qna/local_reranking.py +++ b/comps/reranks/video-rag-qna/local_reranking.py @@ -61,7 +61,6 @@ def find_timestamp_from_video(metadata_list, video): input_datatype=SearchedMultimodalDoc, output_datatype=LVMVideoDoc, ) - @register_statistics(names=["opea_service@reranking_visual_rag"]) def reranking(input: SearchedMultimodalDoc) -> LVMVideoDoc: start = time.time() From bcea36d99c5da7dac34b483a32f4906b179171ec Mon Sep 17 00:00:00 2001 From: BaoHuiling Date: Thu, 29 Aug 2024 14:13:43 +0800 Subject: [PATCH 14/15] add rerank algo desc in readme Signed-off-by: BaoHuiling --- comps/reranks/video-rag-qna/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/comps/reranks/video-rag-qna/README.md b/comps/reranks/video-rag-qna/README.md index fc97faa84..c74165968 100644 --- a/comps/reranks/video-rag-qna/README.md +++ b/comps/reranks/video-rag-qna/README.md @@ -2,6 +2,8 @@ This is a Docker-based microservice that do result rerank for VideoRAGQnA use case. Local rerank is used rather than rerank model. +For the `VideoRAGQnA` usecase, during the data preparation phase, frames are extracted from videos and stored in a vector database. To identify the most relevant video, we count the occurrences of each video source among the retrieved data with rerank function `get_top_doc`. This sorts the video as a descending list of names, ranked by their degree of match with the query. Then we could send the `top_n` videos to the downstream LVM. + # 🚀1. Start Microservice with Docker ## 1.1 Build Images From a1023da0304f8022919113835cd1d6e942f23784 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 06:14:06 +0000 Subject: [PATCH 15/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- comps/reranks/video-rag-qna/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comps/reranks/video-rag-qna/README.md b/comps/reranks/video-rag-qna/README.md index c74165968..9edfe4118 100644 --- a/comps/reranks/video-rag-qna/README.md +++ b/comps/reranks/video-rag-qna/README.md @@ -2,7 +2,7 @@ This is a Docker-based microservice that do result rerank for VideoRAGQnA use case. Local rerank is used rather than rerank model. -For the `VideoRAGQnA` usecase, during the data preparation phase, frames are extracted from videos and stored in a vector database. To identify the most relevant video, we count the occurrences of each video source among the retrieved data with rerank function `get_top_doc`. This sorts the video as a descending list of names, ranked by their degree of match with the query. Then we could send the `top_n` videos to the downstream LVM. +For the `VideoRAGQnA` usecase, during the data preparation phase, frames are extracted from videos and stored in a vector database. To identify the most relevant video, we count the occurrences of each video source among the retrieved data with rerank function `get_top_doc`. This sorts the video as a descending list of names, ranked by their degree of match with the query. Then we could send the `top_n` videos to the downstream LVM. # 🚀1. Start Microservice with Docker