Merge pull request #25 from ddlBoJack/ygr_avsr

Merge AVSR to SLAM pipeline

scripts/finetune_avsr.sh

@@ -1,28 +1,107 @@
 #!/bin/bash
 # export PYTHONPATH=/root/whisper:$PYTHONPATH
-export CUDA_VISIBLE_DEVICES=0
-export CUDA_LAUNCH_BLOCKING=1
+export PYTHONPATH=/root/fairseq:$PYTHONPATH
+export CUDA_VISIBLE_DEVICES=0,1,2,3
+# export CUDA_LAUNCH_BLOCKING=1
+export OMP_NUM_THREADS=1
+
+# debug setting for multiple gpus
+# export NCCL_DEBUG=INFO
+# export NCCL_DEBUG_SUBSYS=ALL
+# export TORCH_DISTRIBUTED_DEBUG=INFO
 
 cd /root/SLAM-LLM
 
-audio_encoder_path=/home/oss/maziyang.mzy/models/AudioMAE/finetuned.pth
-speech_encoder_path=/home/oss/maziyang.mzy/models/Whisper/base.pt
-llm_path=/home/oss/zhifu.gzf/ckpt/Llama-2-7b-hf
-output_dir=/nfs/maziyang.mzy/models/llama-2-hf-finetune
+# speech_encoder_path= TODO!
+
+
+llm_path=/nfs/maziyang.mzy/models/vicuna-7b-v1.5
+# llm_path=/nfs/maziyang.mzy/models/vicuna-13b-v1.5
+
+output_dir=/nfs/yangguanrou.ygr/vicuna-13b-v1.5-finetune-avsr-20230115
 
 # -m debugpy --listen 5678 --wait-for-client
+if [[ $CUDA_VISIBLE_DEVICES != *","* ]]; then
 python src/llama_recipes/pipeline/finetune.py \
 --model_name avsr \
---use_peft --peft_method lora \
---quantization \
---llm_name llama-2-7b-hf \
+--freeze_encoder \
+--freeze_llm \
+--llm_name vicuna-13b-v1.5 \
+--llm_path $llm_path \
+--llm_dim 4096 \
+--encoder_name moco_wav2vec2 \
+--encoder_ds_rate 2 \
+--encoder_dim 512 \
+--encoder_projector linear \
+--encoder_projector_ds_rate 5 \
+--dataset avsr_dataset \
+--avsr_dataset.file src/llama_recipes/datasets/avsr_dataset.py:get_audio_dataset \
+--batching_strategy custom \
+--num_epochs 20 \
+--batch_size_training 6 \
+--val_batch_size 2 \
+--num_workers_dataloader 2 \
+--lr 1e-4 \
+--output_dir $output_dir \
+--metric acc \
+--log_file "/root/SLAM-LLM/log/second_try.log" \
+--use_wandb \
+--wandb_dir $output_dir \
+--wandb_entity_name yanghaha \
+--wandb_project_name slam-llm \
+--wandb_exp_name avsr \
+--log_interval 5 \
+
+else
+torchrun \
+--nnodes 1 \
+--nproc_per_node 4 \
+src/llama_recipes/pipeline/finetune.py \
+--model_name avsr \
+--freeze_encoder \
+--freeze_llm \
+--use_fp16 \
+--enable_fsdp \
+--llm_name vicuna-13b-v1.5 \
 --llm_path $llm_path \
---encoder_name whisper \
---encoder_path $speech_encoder_path \
+--llm_dim 4096 \
+--encoder_name moco_wav2vec2 \
+--encoder_ds_rate 2 \
+--encoder_dim 512 \
 --encoder_projector linear \
+--encoder_projector_ds_rate 5 \
 --dataset avsr_dataset \
 --avsr_dataset.file src/llama_recipes/datasets/avsr_dataset.py:get_audio_dataset \
 --batching_strategy custom \
---num_epochs 1 \
+--num_epochs 20 \
 --batch_size_training 2 \
---output_dir $output_dir 
+--val_batch_size 2 \
+--num_workers_dataloader 2 \
+--lr 1e-4 \
+--output_dir $output_dir \
+--metric acc \
+--log_file "/root/SLAM-LLM/log/second_try.log" \
+--use_wandb \
+--wandb_dir $output_dir \
+--wandb_entity_name yanghaha \
+--wandb_project_name slam-llm \
+--wandb_exp_name avsr \
+--log_interval 5 \
+# --peft_ckpt "/nfs/maziyang.mzy/exps/llama-2-hf-finetune-asr-ds5-proj2048-lr1e-5-whisper-prompt-padding30-20231228/asr/4" \
+# --ckpt_path "/nfs/maziyang.mzy/exps/llama-2-hf-finetune-asr-ds5-proj2048-lr1e-5-whisper-prompt-padding30-20231228/asr/4/model.pt" \
+# --use_peft --peft_method lora \
+# --master_port=29501 \
+fi
+
+# {"key": "1001-134707-0000_ASR", "prompt": "<ASR>", "source": "/cpfs01/shared/Group-speech/beinian.lzr/data/open_data/librispeech_audio/audio/se_librispeech_1001-134707-0000.wav", "target": "1 little recks the laborer. How near his work is holding him to God, The loving laborer through space and time, after all, not to create, only or found only.", "target_len": 157, "source_len": 1581, "text-type": "Transcribe", "audio_language": "en", "text_language": "en", "task-type": "<ASR>"}
+# {"key": "1688-142285-0005", "prompt": "<ASR>", "source": "/nfs/beinian.lzr/workspace/datasets/data/16k/opendata/librispeech/test_other/wav/1688-142285-0005.wav", "target": "YOU WHO WERE ALWAYS ACCUSING PEOPLE OF BEING SHOPPY AT HELSTONE", "target_len": 11, "source_len": 220, "text-type": "Transcribe", "audio_language": "en", "text_language": "en", "task-type": "<ASR>"}
+
+
+
+# 没用 encoder_ds_rate
+
+# 1.15
+
+# 7b batch size 开到2 ok的
+
+#  6 2 0 可以

scripts/finetune_avsr_debug.sh

@@ -1,32 +1,104 @@
 #!/bin/bash
 # export PYTHONPATH=/root/whisper:$PYTHONPATH
+export PYTHONPATH=/root/fairseq:$PYTHONPATH
 export CUDA_VISIBLE_DEVICES=0
-export CUDA_LAUNCH_BLOCKING=1
+# export CUDA_LAUNCH_BLOCKING=1
+export OMP_NUM_THREADS=1
+
+# debug setting for multiple gpus
+# export NCCL_DEBUG=INFO
+# export NCCL_DEBUG_SUBSYS=ALL
+# export TORCH_DISTRIBUTED_DEBUG=INFO
 
 cd /root/SLAM-LLM
 
-audio_encoder_path=/home/oss/maziyang.mzy/models/AudioMAE/finetuned.pth
-speech_encoder_path=/home/oss/maziyang.mzy/models/Whisper/base.pt
+speech_encoder_path=/nfs/zhifu.gzf/ckpt/Whisper/large-v2.pt
+# speech_encoder_path=/nfs/maziyang.mzy/models/Whisper/large-v2-qwen.pt
+
+llm_path=/nfs/maziyang.mzy/models/vicuna-7b-v1.5
+# llm_path=/nfs/maziyang.mzy/models/vicuna-13b-v1.5
+
+output_dir=/nfs/maziyang.mzy/exps/vicuna-7b-v1.5-finetune-asr-ds5-proj2048-lr1e-4-whisper-prompt-paddingr-20240112
 
-llm_path=/nfs/zhifu.gzf/ckpt/Llama-2-7b-hf             #/home/oss/zhifu.gzf/ckpt/Llama-2-7b-hf
-output_dir=/nfs/yangguanrou.ygr/ckpts/llama-2-hf-finetune   #/home/oss/yangguanrou.ygr/ckpts/llama-2-hf-finetune
+# -m debugpy --listen 5678 --wait-for-client
+if [[ $CUDA_VISIBLE_DEVICES != *","* ]]; then
+python -m debugpy --listen 5678 --wait-for-client src/llama_recipes/pipeline/finetune.py \
+--model_name asr \
+--freeze_encoder \
+--freeze_llm \
+--llm_name vicuna-13b-v1.5 \
+--llm_path $llm_path \
+--llm_dim 4096 \
+--encoder_name whisper \
+--encoder_ds_rate 2 \
+--encoder_path $speech_encoder_path \
+--encoder_dim 1280 \
+--encoder_projector linear \
+--encoder_projector_ds_rate 5 \
+--dataset speech_dataset \
+--speech_dataset.train_data_path /nfs/maziyang.mzy/data/librispeech/librispeech_train_960h.jsonl \
+--speech_dataset.val_data_path /nfs/maziyang.mzy/data/librispeech/librispeech_dev_other_filtered.jsonl \
+--batching_strategy custom \
+--num_epochs 100 \
+--batch_size_training 4 \
+--val_batch_size 4 \
+--num_workers_dataloader 4 \
+--lr 1e-4 \
+--output_dir $output_dir \
+--metric acc \
+# --log_file $output_dir/test.log \
+# --use_wandb \
+# --wandb_dir $output_dir \
+# --wandb_entity_name zym22 \
+# --wandb_project_name slam-llm \
+# --wandb_exp_name test \
+# --log_interval 5 \
+# --ckpt_path "/nfs/maziyang.mzy/exps/llama-2-hf-finetune-asr-ds5-proj2048-lr1e-5-whisper-lora-prompt/asr/5/model.pt" \
+# --peft_ckpt "/nfs/maziyang.mzy/exps/llama-2-hf-finetune-asr-ds5-proj2048-lr1e-5-whisper-lora-prompt/asr/5" \
+# --use_peft --peft_method lora \
 
-# -m debugpy --listen 5680 --wait-for-client
-python -m debugpy --listen 5680 --wait-for-client src/llama_recipes/pipeline/finetune.py \
---model_name avsr \
---use_peft --peft_method lora \
---quantization \
---llm_name llama-2-7b-hf \
+else
+torchrun \
+--nnodes 1 \
+--nproc_per_node 4 \
+src/llama_recipes/pipeline/finetune.py \
+--model_name asr \
+--freeze_encoder \
+--freeze_llm \
+--use_fp16 \
+--enable_fsdp \
+--llm_name vicuna-7b-v1.5 \
 --llm_path $llm_path \
+--llm_dim 4096 \
 --encoder_name whisper \
+--encoder_ds_rate 2 \
 --encoder_path $speech_encoder_path \
+--encoder_dim 1280 \
 --encoder_projector linear \
---dataset avsr_dataset \
---avsr_dataset.file src/llama_recipes/datasets/avsr_dataset.py:get_audio_dataset \
+--encoder_projector_ds_rate 5 \
+--dataset speech_dataset \
+--speech_dataset.train_data_path /nfs/maziyang.mzy/data/librispeech/librispeech_train_960h.jsonl \
+--speech_dataset.val_data_path /nfs/maziyang.mzy/data/librispeech/librispeech_dev_other_filtered.jsonl \
 --batching_strategy custom \
---num_epochs 1 \
---batch_size_training 2 \
+--num_epochs 100 \
+--batch_size_training 6 \
+--val_batch_size 6 \
+--num_workers_dataloader 4 \
+--lr 1e-4 \
 --output_dir $output_dir \
---stepSize 10 \
---log_file "/root/SLAM-LLM/log/test.log" \
---valid_subset "LRS3/val_debug.txt" \
+--metric acc \
+--log_file /$output_dir/train.log \
+--use_wandb \
+--wandb_dir $output_dir \
+--wandb_entity_name zym22 \
+--wandb_project_name slam-llm \
+--wandb_exp_name test \
+--log_interval 5 \
+# --peft_ckpt "/nfs/maziyang.mzy/exps/llama-2-hf-finetune-asr-ds5-proj2048-lr1e-5-whisper-prompt-padding30-20231228/asr/4" \
+# --ckpt_path "/nfs/maziyang.mzy/exps/llama-2-hf-finetune-asr-ds5-proj2048-lr1e-5-whisper-prompt-padding30-20231228/asr/4/model.pt" \
+# --use_peft --peft_method lora \
+# --master_port=29501 \
+fi
+
+# {"key": "1001-134707-0000_ASR", "prompt": "<ASR>", "source": "/cpfs01/shared/Group-speech/beinian.lzr/data/open_data/librispeech_audio/audio/se_librispeech_1001-134707-0000.wav", "target": "1 little recks the laborer. How near his work is holding him to God, The loving laborer through space and time, after all, not to create, only or found only.", "target_len": 157, "source_len": 1581, "text-type": "Transcribe", "audio_language": "en", "text_language": "en", "task-type": "<ASR>"}
+# {"key": "1688-142285-0005", "prompt": "<ASR>", "source": "/nfs/beinian.lzr/workspace/datasets/data/16k/opendata/librispeech/test_other/wav/1688-142285-0005.wav", "target": "YOU WHO WERE ALWAYS ACCUSING PEOPLE OF BEING SHOPPY AT HELSTONE", "target_len": 11, "source_len": 220, "text-type": "Transcribe", "audio_language": "en", "text_language": "en", "task-type": "<ASR>"}

scripts/finetune_avsr_vicuna_debug_0113.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# export PYTHONPATH=/root/whisper:$PYTHONPATH
+export PYTHONPATH=/root/fairseq:$PYTHONPATH
+export CUDA_VISIBLE_DEVICES=1
+# export CUDA_LAUNCH_BLOCKING=1
+export OMP_NUM_THREADS=1
+
+# debug setting for multiple gpus
+# export NCCL_DEBUG=INFO
+# export NCCL_DEBUG_SUBSYS=ALL
+# export TORCH_DISTRIBUTED_DEBUG=INFO
+
+cd /root/SLAM-LLM
+
+# speech_encoder_path= TODO!
+
+
+llm_path=/nfs/maziyang.mzy/models/vicuna-7b-v1.5
+# llm_path=/nfs/maziyang.mzy/models/vicuna-13b-v1.5
+
+output_dir=/nfs/yangguanrou.ygr/vicuna-7b-v1.5-finetune-avsr
+
+# -m debugpy --listen 5678 --wait-for-client
+if [[ $CUDA_VISIBLE_DEVICES != *","* ]]; then
+python -m debugpy --listen 5679 --wait-for-client src/llama_recipes/pipeline/finetune.py \
+--model_name avsr \
+--freeze_encoder \
+--freeze_llm \
+--llm_name vicuna-13b-v1.5 \
+--llm_path $llm_path \
+--llm_dim 4096 \
+--encoder_name moco_wav2vec2 \
+--encoder_ds_rate 2 \
+--encoder_dim 512 \
+--encoder_projector linear \
+--encoder_projector_ds_rate 5 \
+--dataset avsr_dataset \
+--avsr_dataset.file src/llama_recipes/datasets/avsr_dataset.py:get_audio_dataset \
+--batching_strategy custom \
+--num_epochs 1 \
+--batch_size_training 2 \
+--num_workers_dataloader 2 \
+--lr 1e-4 \
+--output_dir $output_dir \
+--metric acc \
+--log_file "/root/SLAM-LLM/log/first_try.log" \
+
+
+# --avsr_dataset.file src/llama_recipes/datasets/avsr_dataset.py:get_audio_dataset \
+
+
+# --encoder_path $speech_encoder_path \   #TODO!
+# --encoder_dim 1280 \   #TODO!

src/llama_recipes/configs/datasets.py

@@ -70,12 +70,10 @@ class avsr_dataset:
     noiseProb: float = 0.
     noiseSNR: float = 5
     stepSize: int = 16384
-    # charToIx={" ": 1, "'": 22, "1": 30, "0": 29, "3": 37, "2": 32, "5": 34, "4": 38, "7": 36, "6": 35, "9": 31, "8": 33, "A": 5, "C": 17,
-    #                      "B": 20, "E": 2, "D": 12, "G": 16, "F": 19, "I": 6, "H": 9, "K": 24, "J": 25, "M": 18, "L": 11, "O": 4, "N": 7, "Q": 27,
-    #                      "P": 21, "S": 8, "R": 10, "U": 13, "T": 3, "W": 15, "V": 23, "Y": 14, "X": 26, "Z": 28, "<EOS>": 39}
     charToIx : str = "x"   #应该没用了  TypeError: Object of type NotImplementedType is not JSON serializable 但这个是上面的问题
     modal: str = "AV"
     pretrain_subset: str = "LRS3/pretrain.txt"
     train_subset: str = "LRS3/train.txt"
     valid_subset: str = "LRS3/val.txt"
     test_subset: str = "LRS3/test.txt"
+    reqInpLen: str = 80