diff --git a/lmms_eval/models/gemini_api.py b/lmms_eval/models/gemini_api.py
index 69f520c1..3fb1d39f 100644
--- a/lmms_eval/models/gemini_api.py
+++ b/lmms_eval/models/gemini_api.py
@@ -1,6 +1,8 @@
 import io
 import json
 import os
+import pathlib
+import re
 import time
 from typing import List, Tuple
 
@@ -39,8 +41,9 @@ def __init__(
         model_version: str = "gemini-1.5-pro",
         # modality: str = "image",
         timeout: int = 120,
-        continual_mode: bool = False,
+        continual_mode: bool = True,
         response_persistent_folder: str = "./logs/gemini_persistent_folder",
+        interleave: bool = False,
         # We will cache the Gemini API response in this path and use it for future requests
         **kwargs,
     ) -> None:
@@ -49,6 +52,7 @@ def __init__(
         self.timeout = timeout
         self.model = genai.GenerativeModel(model_version)
         self.continual_mode = continual_mode
+        self.interleave = interleave
         # if self.continual_mode and response_persistent_folder is None:
         #     raise ValueError("Continual mode requires a persistent path for the response. We will cache the Gemini API response in this path and use it for future requests. Please provide a valid path.")
         if self.continual_mode:
@@ -132,6 +136,20 @@ def convert_modality(self, images):
                     eval_logger.error(f"Error converting video: {str(e)}")
         return images
 
+    def construct_interleaved_input(self, content, media):
+        pattern = r"<media_(\d+)>"
+        parts = re.split(pattern, content)
+        result = []
+        for i, part in enumerate(parts):
+            if i % 2 == 0:
+                if part == "":
+                    continue
+                result.append(part)
+            else:
+                result.append(media[int(part)])
+
+        return result
+
     def generate_until(self, requests) -> List[str]:
         res = []
         pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
@@ -163,7 +181,10 @@ def get_uuid(task, split, doc_id):
             visuals = self.flatten(visuals)
             visuals = self.convert_modality(visuals)
 
-            message = [contexts] + visuals
+            if self.interleave:
+                message = self.construct_interleaved_input(contexts, visuals)
+            else:
+                message = [contexts] + visuals
 
             for attempt in range(5):
                 try:
@@ -213,3 +234,72 @@ def generate_until_multi_round(self, requests) -> List[str]:
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         # TODO
         assert False, "Gemini API not support"
+
+    def get_image_audio_text_interleaved_messsage(self, image_path, audio_path, question):
+        # image_path for list of image path
+        # audio_path for list of audio path
+        # question for question
+
+        # fixed image token and no audio in text
+        for index in range(1, 1 + len(image_path)):
+            question = question.replace(f"[img{index}]", "<image>")
+        for index in range(1, 1 + len(audio_path)):
+            question = question.replace(f"[audio{index}]", "<audio>")
+
+        text = question
+
+        info_list = []
+        image_counter = 0
+        audio_counter = 0
+        for part in re.split(r"(<image>|<audio>)", text):
+            if part == "<image>":
+                info_list.append(Image.open(image_path[image_counter]))
+                image_counter += 1
+            elif part == "<audio>":
+                info_list.append({"mime_type": "audio/wav", "data": pathlib.Path(audio_path[audio_counter]).read_bytes()})
+                audio_counter += 1
+            else:
+                if part == " ":
+                    continue
+                info_list.append(part)
+
+        return info_list
+
+    def get_video_audio_text_interleaved_message(self, video_path, audio_path, question):
+        # image_path for list of image path
+        # audio_path for list of audio path
+        # question for question
+
+        # fixed video token and no audio in text
+        for index in range(1, 1 + len(video_path)):
+            question = question.replace(f"[video{index}]", "<video>")
+        for index in range(1, 1 + len(audio_path)):
+            question = question.replace(f"[audio{index}]", "<audio>")
+
+        text = question
+
+        info_list = []
+        video_counter = 0
+        audio_counter = 0
+        for part in re.split(r"(<video>|<audio>)", text):
+            if part == "<video>":
+                current_video_file_name = video_path[video_counter]
+                current_video_file = genai.upload_file(path=current_video_file_name)
+                while current_video_file.state.name == "processing":
+                    print("uploading file")
+                    time.sleep(5)
+                    current_video_file = genai.get_file(current_video_file.name)
+                if current_video_file.state.name == "FAILED":
+                    print("uploading file failed, next question")
+                    return 0
+                info_list.append(current_video_file)
+                video_counter += 1
+            elif part == "<audio>":
+                info_list.append({"mime_type": "audio/wav", "data": pathlib.Path(audio_path[audio_counter]).read_bytes()})
+                audio_counter += 1
+            else:
+                if part == " ":
+                    continue
+                info_list.append(part)
+
+        return info_list
diff --git a/lmms_eval/models/text.ipynb b/lmms_eval/models/text.ipynb
new file mode 100644
index 00000000..de40cc2c
--- /dev/null
+++ b/lmms_eval/models/text.ipynb
@@ -0,0 +1,156 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re\n",
+    "\n",
+    "def split_media_tags(content):\n",
+    "    # 用正则表达式匹配 <media数字>\n",
+    "    # (\\d+) 捕获组用来提取数字\n",
+    "    pattern = r'<media(\\d+)>'\n",
+    "    \n",
+    "    # 用 split 方法分割文本,同时保留匹配到的数字\n",
+    "    # re.split 会返回一个列表,包含分割后的文本和匹配到的捕获组\n",
+    "    parts = re.split(pattern, content)\n",
+    "    \n",
+    "    # 处理结果列表,将数字转换为整型\n",
+    "    result = []\n",
+    "    for i, part in enumerate(parts):\n",
+    "        if i % 2 == 0:  # 偶数索引是文本\n",
+    "            result.append(part)\n",
+    "        else:  # 奇数索引是数字\n",
+    "            result.append(int(part))\n",
+    "            \n",
+    "    return result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['', 1, 'world', 2, '!']"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "split_media_tags('<media1>world<media2>!')  # ['hello', 1, 'world', 2, '!']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Input: Select the instrument represented in images that corresponds to the audio [audio1] from [img1] [img2] [img3] [img4].\n",
+      "Output: ['Select the instrument represented in images that corresponds to the audio ', ('audio', 1), ' from ', ('img', 1), ' ', ('img', 2), ' ', ('img', 3), ' ', ('img', 4), '.']\n",
+      "\n",
+      "Input: [video1] is a great video\n",
+      "Output: [('video', 1), ' is a great video']\n",
+      "\n",
+      "Input: Compare [img1] and [img2]\n",
+      "Output: ['Compare ', ('img', 1), ' and ', ('img', 2)]\n",
+      "\n",
+      "Input: Listen to [audio1] and watch [video1]\n",
+      "Output: ['Listen to ', ('audio', 1), ' and watch ', ('video', 1)]\n",
+      "\n",
+      "Input: [img1] at the beginning and [img2] at the end\n",
+      "Output: [('img', 1), ' at the beginning and ', ('img', 2), ' at the end']\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import re\n",
+    "\n",
+    "def split_media_tags(content):\n",
+    "    # 匹配 [类型数字] 格式的标签\n",
+    "    # 捕获组 1 捕获类型 (audio|video|img)\n",
+    "    # 捕获组 2 捕获数字\n",
+    "    pattern = r'\\[(audio|video|img)(\\d+)\\]'\n",
+    "    \n",
+    "    # 用 finditer 找到所有匹配\n",
+    "    matches = list(re.finditer(pattern, content))\n",
+    "    if not matches:\n",
+    "        return [content]\n",
+    "    \n",
+    "    result = []\n",
+    "    last_end = 0\n",
+    "    \n",
+    "    for match in matches:\n",
+    "        # 添加标签之前的文本（如果有）\n",
+    "        if match.start() > last_end:\n",
+    "            result.append(content[last_end:match.start()])\n",
+    "            \n",
+    "        # 添加标签信息为元组 (类型, 数字)\n",
+    "        media_type = match.group(1)  # audio, video 或 img\n",
+    "        media_num = int(match.group(2))  # 数字\n",
+    "        result.append((media_type, media_num))\n",
+    "        \n",
+    "        last_end = match.end()\n",
+    "    \n",
+    "    # 添加最后一个标签之后的文本（如果有）\n",
+    "    if last_end < len(content):\n",
+    "        result.append(content[last_end:])\n",
+    "    \n",
+    "    return result\n",
+    "\n",
+    "# 测试\n",
+    "test_cases = [\n",
+    "    \"Select the instrument represented in images that corresponds to the audio [audio1] from [img1] [img2] [img3] [img4].\",\n",
+    "    \"[video1] is a great video\",\n",
+    "    \"Compare [img1] and [img2]\",\n",
+    "    \"Listen to [audio1] and watch [video1]\",\n",
+    "    \"[img1] at the beginning and [img2] at the end\",\n",
+    "]\n",
+    "\n",
+    "for test in test_cases:\n",
+    "    print(f\"Input: {test}\")\n",
+    "    print(f\"Output: {split_media_tags(test)}\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "av-odyssey",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/lmms_eval/tasks/av_odyssey/av_odyssey.yaml b/lmms_eval/tasks/av_odyssey/av_odyssey.yaml
new file mode 100644
index 00000000..926cf451
--- /dev/null
+++ b/lmms_eval/tasks/av_odyssey/av_odyssey.yaml
@@ -0,0 +1,24 @@
+dataset_path: AV-Odyssey/AV_Odyssey_Bench_LMMs_Eval
+dataset_kwargs:
+  token: True
+  cache_dir: AV_Odyssey
+  video: True
+task: "av_odyssey"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.av_odyssey_doc_to_visual
+doc_to_text: !function utils.av_odyssey_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+# The return value of process_results will be used by metrics
+process_results: !function utils.av_odyssey_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: av_odyssey_score
+    aggregation: !function utils.av_odyssey_aggregate_results
+    higher_is_better: true
\ No newline at end of file
diff --git a/lmms_eval/tasks/av_odyssey/utils.py b/lmms_eval/tasks/av_odyssey/utils.py
new file mode 100644
index 00000000..754e702c
--- /dev/null
+++ b/lmms_eval/tasks/av_odyssey/utils.py
@@ -0,0 +1,264 @@
+import datetime
+import os
+import random
+import re
+from collections import defaultdict
+from pathlib import Path
+
+import numpy as np
+import yaml
+from loguru import logger as eval_logger
+from PIL import Image
+
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+
+dir_name = os.path.dirname(os.path.abspath(__file__))
+
+eval_type_dict = {
+    "Subfield": [
+        "Timbre",
+        "Tone",
+        "Melody",
+        "Space",
+        "Time",
+        "Hallucination",
+        "Intricacy",
+    ],
+}
+
+with open(Path(__file__).parent / "av_odyssey.yaml", "r") as f:
+    raw_data = f.readlines()
+    safe_data = []
+    for i, line in enumerate(raw_data):
+        # remove function definition since yaml load cannot handle it
+        if "!function" not in line:
+            safe_data.append(line)
+
+    config = yaml.safe_load("".join(safe_data))
+
+hf_home = os.getenv("HF_HOME", "~/.cache/huggingface/")
+cache_dir = os.path.join(hf_home, config["dataset_kwargs"]["cache_dir"])
+
+question_prompt = "Answer with the option's letter from the given choices directly."
+
+
+def split_media_tags(content):
+    pattern = r"\[(audio|video|img)(\d+)\]"
+
+    matches = list(re.finditer(pattern, content))
+    if not matches:
+        return [content]
+
+    result = []
+    last_end = 0
+
+    for match in matches:
+        if match.start() > last_end:
+            result.append(content[last_end : match.start()])
+
+        media_type = match.group(1)
+        media_num = int(match.group(2))
+        result.append((media_type, media_num))
+
+        last_end = match.end()
+
+    if last_end < len(content):
+        result.append(content[last_end:])
+
+    return result
+
+
+def av_odyssey_doc_to_visual(doc):
+    audio_data = []
+    image_data = []
+    video_data = []
+    result = []
+
+    # 处理 image 类型数据
+    if "image" in doc["data_type"]:
+        for relative_path in doc["image_path"]:
+            abs_path = os.path.join(cache_dir, relative_path)
+            if os.path.exists(abs_path):
+                image_data.append(abs_path)  # 保留路径以供后续处理
+            else:
+                print(f"Image path does not exist: {abs_path}")
+
+    # 处理 video 类型数据
+    elif "video" in doc["data_type"]:
+        for relative_path in doc["video_path"]:
+            abs_path = os.path.join(cache_dir, relative_path)
+            if os.path.exists(abs_path):
+                video_data.append(abs_path)  # 保留路径以供后续处理
+            else:
+                print(f"Video path does not exist: {abs_path}")
+
+    # 处理 audio 类型数据
+    for relative_path in doc["audio_path"]:
+        abs_path = os.path.join(cache_dir, relative_path)
+        if os.path.exists(abs_path):
+            audio_data.append(abs_path)  # 保留路径以供后续处理
+        else:
+            print(f"Audio path does not exist: {abs_path}")
+
+    question = get_text(doc)
+    for q in question:
+        if isinstance(q, str):
+            continue
+        else:
+            media_type, media_num = q
+            media_num = media_num - 1
+            if media_type == "audio":
+                result.append(audio_data[media_num])
+            elif media_type == "video":
+                result.append(video_data[media_num])
+            elif media_type == "img":
+                result.append(image_data[media_num])
+
+    return result
+
+
+def get_text(doc):
+    question = doc["question"]
+    options = doc["options"]
+    option_text = options[0] + "\n" + options[1] + "\n" + options[2] + "\n" + options[3] + "\n"
+    text = question + "\n" + option_text + question_prompt
+    return split_media_tags(text)
+
+
+def av_odyssey_doc_to_text(doc, lmms_eval_specific_kwargs=None):
+    text = get_text(doc)
+    id = 0
+    result = []
+    for t in text:
+        if isinstance(t, str):
+            result.append(t)
+        else:
+            result.append(f"<media_{id}>")
+            id += 1
+    return "".join(result)
+
+
+def parse_multi_choice_response(response, all_choices, index2ans):
+    """
+    Parse the prediction from the generated response.
+    Return the predicted index e.g., A, B, C, D.
+    """
+    for char in [",", ".", "!", "?", ";", ":", "'"]:
+        response = response.strip(char)
+    response = " " + response + " "  # add space to avoid partial match
+
+    index_ans = True
+    ans_with_brack = False
+    candidates = []
+    for choice in all_choices:  # e.g., (A) (B) (C) (D)
+        if f"{choice}" in response:
+            candidates.append(choice)
+            ans_with_brack = True
+
+    if len(candidates) == 0:
+        for choice in all_choices:  # e.g., A B C D
+            if f" {choice} " in response:
+                candidates.append(choice)
+
+    # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example
+    if len(candidates) == 0 and len(response.split()) > 5:
+        for index, ans in index2ans.items():
+            if ans.lower() in response.lower():
+                candidates.append(index)
+                index_ans = False  # it's content ans.
+
+    if len(candidates) == 0:  # still not get answer, randomly choose one.
+        # pred_index = random.choice(all_choices)
+        pred_index = "A"
+    elif len(candidates) > 1:
+        start_indexes = []
+        if index_ans:
+            if ans_with_brack:
+                for can in candidates:
+                    index = response.rfind(f"({can})")
+                    start_indexes.append(index)  # -1 will be ignored anyway
+                # start_indexes = [generated_response.index(f'({can})') for can in candidates]
+            else:
+                for can in candidates:
+                    index = response.rfind(f" {can} ")
+                    start_indexes.append(index)
+        else:
+            for can in candidates:
+                index = response.lower().rfind(index2ans[can].lower())
+                start_indexes.append(index)
+        # get the last one
+        pred_index = candidates[np.argmax(start_indexes)]
+    else:  # if only one candidate, use it.
+        pred_index = candidates[0]
+
+    return pred_index
+
+
+def av_odyssey_process_results(doc, results):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name (in this case av_odyssey score), value: metric value
+    """
+    pred = results[0]
+    options = doc["options"]
+    option_list = {"A": options[0][3:], "B": options[1][3:], "C": options[2][3:], "D": options[3][3:]}
+    answer = parse_multi_choice_response(pred, ["A", "B", "C", "D"], option_list)
+    gt_answer = doc["answer"]
+    assert answer in ["A", "B", "C", "D"]
+    assert gt_answer in ["A", "B", "C", "D"]
+    score = 1.0 if answer == gt_answer else 0.0
+    category = doc["subfield"]
+    key_name = "av_odyssey_score"
+    # Note: the key name here is very important. It decides which aggregation function will receive the results
+    # We note down the question id/category to help us aggregate the results later
+    return {key_name: {"question_id": doc["question_id"], "category": category, "score": score}}
+
+
+def av_odyssey_aggregate_results(results):
+    """
+    Args:
+        results: a list of values returned by process_results
+    Returns:
+        A score
+    """
+    category2score = defaultdict(dict)
+    for result in results:
+        question_id = result["question_id"]
+        score = result["score"]
+        category = result["category"]
+        if question_id not in category2score[category]:
+            category2score[category][question_id] = []
+        category2score[category][question_id].append(score)
+
+    # 计算每个 category 的平均分
+    category_avg_scores = {}
+    total_score = 0
+    total_questions = 0
+
+    # 遍历所有 category 来计算每个 category 的平均分
+    for category, questions in category2score.items():
+        # import pdb
+        # pdb.set_trace()
+        category_total = 0  # 计算所有问题的总分
+        for question_id, score in questions.items():
+            category_total += score[0]  # 累加所有问题的平均分
+        category_avg_scores[category] = category_total / len(questions) * 100.0  # 当前类别的平均分
+
+        total_score += category_total  # 累加所有类别的问题总分
+        total_questions += len(questions)  # 累加所有问题的数量
+
+    # 计算所有问题的平均分（按问题的总数来平均）
+    overall_avg_score = total_score / total_questions * 100.0
+
+    # 输出每个 category 的平均分
+    print("Average scores per category:")
+    for category, avg_score in category_avg_scores.items():
+        print(f"{category}: {avg_score:.2f}")
+
+    # 输出所有问题的平均分
+    print(f"Overall average score (across all questions): {overall_avg_score:.2f}")
+
+    return overall_avg_score