ds_config_sft.json

{
    "train_batch_size": "auto",
    "train_micro_batch_size_per_gpu": "auto",
    "gradient_accumulation_steps": "auto",
    "optimizer": {
      "type": "AdamW",
      "params": {
        "lr": "auto",
        "betas": "auto",
        "eps": "auto",
        "weight_decay": "auto"
      }
    },
    "bf16": {
      "enabled": true,
      "min_loss_scale": 1,
      "opt_level": "O2"
    },
    "scheduler": {
      "type": "WarmupDecayLR",
      "params": {
          "warmup_min_lr": 0,
          "warmup_max_lr": "auto",
          "warmup_num_steps": "auto",
          "total_num_steps": "auto"
      }
      },
    "zero_optimization": {
      "stage": 3,
      "offload_param": {
        "device": "cpu"
      },
      "offload_optimizer": {
        "device": "cpu"
      },
      "allgather_partitions": true,
      "allgather_bucket_size": 5e8,
      "contiguous_gradients": true
    }
  }