Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fine-tune-llm] Add Mistral 7b full param training config #88

Merged
merged 4 commits into from
Feb 24, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions templates/fine-tune-llm/deepspeed_configs/zero_3_mistral_7b.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"fp16": {
"enabled": "auto"
},
"bf16": {
"enabled": true
},
"zero_optimization": {
"stage": 3,
"offload_optimizer": {
"device": "cpu",
"pin_memory": true
},
"offload_param": {
"device": "cpu",
"pin_memory": true
},
"overlap_comm": true,
"contiguous_gradients": true,
"sub_group_size": 1e9,
"reduce_bucket_size": 5e8,
"stage3_prefetch_bucket_size": 5e8,
"stage3_param_persistence_threshold": 1e6,
"stage3_max_live_parameters": 1e9,
"stage3_max_reuse_distance": 1e9,
"stage3_gather_16bit_weights_on_model_save": true,
"round_robin_gradients": true
},
"gradient_accumulation_steps": "auto",
"gradient_clipping": "auto",
"steps_per_print": 10,
"train_batch_size": "auto",
"train_micro_batch_size_per_gpu": "auto",
"wall_clock_breakdown": false
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
model_id: mistralai/Mistral-7B-Instruct-v0.1 # <-- change this to the model you want to fine-tune
train_path: s3://air-example-data/viggo/train.jsonl
valid_path: s3://air-example-data/viggo/valid.jsonl
context_length: 4096 # <-- change this to the context length you want to use
num_devices: 16 # <-- change this to total number of GPUs that you want to use
num_epochs: 1 # <-- change this to the number of epochs that you want to train for
train_batch_size_per_device: 2
eval_batch_size_per_device: 2
learning_rate: 1e-4
num_checkpoints_to_keep: 1
no_gradient_checkpoint: False
dataset_size_scaling_factor: 10000
output_dir: /mnt/local_storage
deepspeed:
config_path: deepspeed_configs/zero_3_mistral_7b.json
flash_attention_2: True
worker_resources:
g5.12xlarge: 1 # <-- this maps to job_compute_configs file's custom_resources so the appropriate nodes can scale up
Loading