117M.json

{
    "n_head": 12,
    "encoder_path": "gs://openwebtext/stuff/encoder",
    "n_vocab": 50257,
    "embed_dropout": 0.1,
    "lr": 0.00025,
    "warmup_steps": 2000,
    "beta1": 0.9,
    "beta2": 0.98,
    "epsilon": 1e-9,
    "opt_name": "adam",
    "weight_decay": 0.01,
    "train_batch_size": 32,
    "attn_dropout": 0.1,
    "train_steps": 10000,
    "eval_steps": 10,
    "max_steps": 500000,
    "data_path": "gs://connors-datasets/openwebtext/",
    "res_dropout": 0.1,
    "predict_batch_size": 1,
    "eval_batch_size": 32,
    "iterations": 500,
    "n_embd": 768,
    "input": "openwebtext",
    "model": "GPT2",
    "model_path": "gs://connors-models/GPT2-117M-test-newinit",
    "n_ctx": 1024,
    "predict_path": "logs/predictions.txt",
    "n_layer": 12,
    "scale_by_depth": true,
    "scale_by_in": true
}