-
Notifications
You must be signed in to change notification settings - Fork 22
/
finetune_stage3.sh
40 lines (38 loc) · 1.22 KB
/
finetune_stage3.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
#!/bin/bash
MODEL_VERSION="vicuna-7b-v1-5"
PROMPT_VERSION=llava_v1
DATA_ROOT=./playground/data
model_size=7b
deepspeed --master_port 12347 llava/train/train_mem.py \
--deepspeed ./scripts/zero2.json \
--model_name_or_path ./checkpoints/vip-llava-$model_size-stage2-ft \
--version $PROMPT_VERSION \
--data_path $DATA_ROOT/vip-llava_stage3_mix.json \
--image_folder $DATA_ROOT \
--vision_tower clip_4layers_336 \
--mm_projector_type mlp2x_gelu \
--mm_vision_select_layer -2 \
--mm_use_im_start_end False \
--mm_use_im_patch_token False \
--image_aspect_ratio pad \
--bf16 True \
--output_dir ./checkpoints/vip-llava-$model_size-stage3-ft \
--num_train_epochs 1 \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 4 \
--gradient_accumulation_steps 1 \
--evaluation_strategy "no" \
--save_strategy "steps" \
--save_steps 50000 \
--save_total_limit 1 \
--learning_rate 2e-7 \
--weight_decay 0. \
--warmup_ratio 0.03 \
--lr_scheduler_type "cosine" \
--logging_steps 1 \
--tf32 True \
--model_max_length 2048 \
--gradient_checkpointing True \
--dataloader_num_workers 4 \
--lazy_preprocess True \
--report_to wandb