{
    "predictor": {
        "attention_bias": false,
        "attention_dropout": 0.0,
        "bos_token_id": 1,
        "pad_token_id": null,
        "dropout": 0.00,
        "eos_token_id": 2,
        "hidden_act": "silu",
        "hidden_size": 4096,
        "initializer_range": 0.02,
        "intermediate_size": 11008,
        "mask_image_logits": false,
        "max_position_embeddings": 10240,
        "mlp_bias": false,
        "num_heads": 32,
        "num_layers": 32,
        "num_key_value_heads": 32,
        "rms_norm_eps": 1e-05,
        "rope_scaling": null,
        "rope_theta": 10000.0,
        "swin_norm": false,
        "tie_word_embeddings": false,
        "vocab_size": 171385,
        "vocabulary_map_path": "./weights/Lumina/Lumina-mGPT-2.0/"
    },
    "z_loss_weight": 1e-5,
    "patch": {
        "use_fsdp1": {
            "sharding_strategy": "fsdp",
            "param_dtype": "bf16",
            "reduce_dtype": "fp32",
            "forward_prefetch": true,
            "backward_prefetch": "backward_pre"
        }
    }
}