[Feat] Add AuroraCap, MovieChat, LLaVA-OneVision-MovieChat (#358)

* Add AuroraCap, MovieChat, LLaVA-OneVision-MovieChat * Add tasks to readme * Add tasks to readme * Modify init * Add instruction
EvolvingLMMs-Lab · Oct 26, 2024 · 60bbec6 · 60bbec6
1 parent 924814c
commit 60bbec6
Show file tree

Hide file tree

Showing 6 changed files with 1,605 additions and 1 deletion.
diff --git a/docs/current_tasks.md b/docs/current_tasks.md
@@ -292,6 +292,17 @@
 
 - [YouCook2](http://youcook2.eecs.umich.edu/) (youcook2_val)
 
+- [MovieChat](https://github.com/rese1f/MovieChat) (moviechat)
+  - MovieChat Global Model (moviechat_global)
+  - MovieChat Breakpoint Model (moviechat_breakpoint)
+
+- [VDC](https://github.com/rese1f/aurora) (vdc)
+  - VDC Camera Caption (camera_test)
+  - VDC Short Caption (short_test)
+  - VDC Background Caption (background_test)
+  - VDC Main Object Caption (main_object_test)
+
+
 ## 4. Text Tasks
 
 - [GSM8K](https://github.com/openai/grade-school-math) (gsm8k)

diff --git a/docs/run_examples.md b/docs/run_examples.md
@@ -433,4 +433,75 @@ accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
     --log_samples \
     --log_samples_suffix $TASK_SUFFIX \
     --output_path ./logs/ 
-```
+```
+
+### MovieChat
+
+```bash
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+python -m pip install torch==2.0.1 torchvision==0.15.2 --extra-index-url https://download.pytorch.org/whl/cu118
+
+git clone https://github.com/rese1f/MovieChat.git
+mv /path/to/MovieChat /path/to/lmms-eval/lmms_eval/models/
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+    --model moviechat \
+    --tasks $TASK \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix $TASK_SUFFIX \
+    --output_path ./logs/ 
+```
+
+### LLaVA-OneVision-MovieChat
+
+```bash
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+git clone https://github.com/rese1f/MovieChat.git
+mv /path/to/MovieChat/MovieChat_OneVision/llava /path/to/lmms-eval/
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+    --model llava_onevision_moviechat \
+    --tasks $TASK \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix $TASK_SUFFIX \
+    --output_path ./logs/ 
+```
+
+### LLaVA-OneVision-MovieChat
+
+```bash
+cd /path/to/lmms-eval
+python3 -m pip install -e .;
+
+git clone https://github.com/rese1f/aurora.git
+mv /path/to/aurora/src/xtuner/xtuner /path/to/lmms-eval/xtuner-aurora
+
+TASK=$1
+echo $TASK
+TASK_SUFFIX="${TASK//,/_}"
+echo $TASK_SUFFIX
+
+accelerate launch --num_processes 8 --main_process_port 12345 -m lmms_eval \
+    --model auroracap \
+    --tasks $TASK \
+    --batch_size 1 \
+    --log_samples \
+    --log_samples_suffix $TASK_SUFFIX \
+    --output_path ./logs/ 
+```
diff --git a/lmms_eval/models/__init__.py b/lmms_eval/models/__init__.py
@@ -11,6 +11,7 @@
 logger.add(sys.stdout, level="WARNING")
 
 AVAILABLE_MODELS = {
+    "auroracap": "AuroraCap",
     "batch_gpt4": "BatchGPT4",
     "claude": "Claude",
     "cogvlm2": "CogVLM2",
@@ -26,12 +27,14 @@
     "llava": "Llava",
     "llava_hf": "LlavaHf",
     "llava_onevision": "Llava_OneVision",
+    "llava_onevision_moviechat": "Llava_OneVision_MovieChat",
     "llava_sglang": "LlavaSglang",
     "llava_vid": "LlavaVid",
     "longva": "LongVA",
     "mantis": "Mantis",
     "minicpm_v": "MiniCPM_V",
     "minimonkey": "MiniMonkey",
+    "moviechat": "MovieChat",
     "mplug_owl_video": "mplug_Owl",
     "phi3v": "Phi3v",
     "qwen_vl": "Qwen_VL",