{
"test_llama2_mcore_agieval_evaluate": [
{
"param": {
"task-data-path":"/data/ci/datasets/eval_data/agieval",
"task":"agieval",
"tensor-model-parallel-size": 8,
"pipeline-model-parallel-size": 1,
"num-layers": 32,
"use-mcore-models": null,
"hidden-size": 4096,
"ffn-hidden-size": 11008,
"num-attention-heads": 32,
"seq-length": 4096,
"max-new-tokens": 1,
"evaluation-batch-size": 1,
"max-position-embeddings": 4096,
"make-vocab-size-divisible-by": 1,
"micro-batch-size": 1,
"normalization": "RMSNorm",
"position-embedding-type": "rope",
"swiglu": null,
"tokenizer-not-use-fast": null,
"untie-embeddings-and-output-weights": null,
"disable-bias-linear": null,
"no-masked-softmax-fusion": null,
"fp16": null,
"no-load-rng": null,
"no-load-optim": null,
"load":"/data/ci/models/llama2/mg/llama2-tp8pp1",
"tokenizer-type":"PretrainedFromHF",
"tokenizer-name-or-path":"/data/ci/models/llama2/hf/llama-2-7b-hf",
"npu-deterministic": null,
"transformer-impl": "local"
}
}
],
"test_llama2_mcore_bbh_evaluate": [
{
"param": {
"task-data-path":"/data/ci/datasets/eval_data/bbh/test",
"task":"bbh",
"tensor-model-parallel-size": 8,
"pipeline-model-parallel-size": 1,
"use-mcore-models": null,
"num-layers": 32,
"hidden-size": 4096,
"ffn-hidden-size": 11008,
"num-attention-heads": 32,
"seq-length": 4096,
"max-new-tokens": 32,
"evaluation-batch-size": 4,
"max-position-embeddings": 4096,
"make-vocab-size-divisible-by": 1,
"micro-batch-size": 1,
"normalization": "RMSNorm",
"position-embedding-type": "rope",
"swiglu": null,
"tokenizer-not-use-fast": null,
"untie-embeddings-and-output-weights": null,
"disable-bias-linear": null,
"no-masked-softmax-fusion": null,
"fp16": null,
"no-load-rng": null,
"no-load-optim": null,
"load":"/data/ci/models/llama2/mg/llama2-tp8pp1",
"tokenizer-type":"PretrainedFromHF",
"tokenizer-name-or-path":"/data/ci/models/llama2/hf/llama-2-7b-hf",
"npu-deterministic": null,
"transformer-impl": "local"
}
}
],
"test_llama2_mcore_humaneval_evaluate": [
{
"param": {
"task-data-path":"/data/ci/datasets/eval_data/human_eval",
"task":"human_eval",
"tensor-model-parallel-size": 8,
"pipeline-model-parallel-size": 1,
"num-layers": 32,
"use-mcore-models": null,
"hidden-size": 4096,
"ffn-hidden-size": 11008,
"num-attention-heads": 32,
"seq-length": 4096,
"max-new-tokens": 200,
"evaluation-batch-size": 1,
"max-position-embeddings": 4096,
"make-vocab-size-divisible-by": 1,
"micro-batch-size": 1,
"normalization": "RMSNorm",
"position-embedding-type": "rope",
"swiglu": null,
"tokenizer-not-use-fast": null,
"untie-embeddings-and-output-weights": null,
"disable-bias-linear": null,
"no-masked-softmax-fusion": null,
"fp16": null,
"no-load-rng": null,
"no-load-optim": null,
"transformer-impl": "local",
"load":"/data/ci/models/llama2/mg/llama2-tp8pp1",
"tokenizer-type":"PretrainedFromHF",
"tokenizer-name-or-path":"/data/ci/models/llama2/hf/llama-2-7b-hf"
}
}
],
"test_qwen2_mcore_needlebench_evaluate": [
{
"param": {
"use-mcore-models": null,
"task-data-path": "/data/ci/datasets/eval_data/NeedleBench/",
"task": "needlebench",
"tensor-model-parallel-size": 4,
"pipeline-model-parallel-size": 1,
"seq-length": 131072,
"max-position-embeddings": 131072,
"max-new-tokens": 64,
"num-layers": 2,
"hidden-size": 3584,
"ffn-hidden-size": 18944,
"num-attention-heads": 28,
"disable-bias-linear": null,
"swiglu": null,
"position-embedding-type": "rope",
"load": "/data/ci/models/qwen2-7b/mg/qwen2-7b-2layer-tp4pp1/",
"normalization": "RMSNorm",
"tokenizer-type": "PretrainedFromHF",
"tokenizer-name-or-path": "/data/ci/models/qwen2-7b/hf/qwen2-7b-hf/",
"tokenizer-not-use-fast": null,
"micro-batch-size": 1,
"use-kv-cache": null,
"no-load-rng": null,
"no-load-optim": null,
"untie-embeddings-and-output-weights": null,
"add-qkv-bias": null,
"exit-on-missing-checkpoint": null,
"make-vocab-size-divisible-by": 1,
"padded-vocab-size": 128000,
"rotary-base": 1000000,
"no-gradient-accumulation-fusion": null,
"attention-softmax-in-fp32": null,
"seed": 42,
"group-query-attention": null,
"num-query-groups": 4,
"max-tokens-to-oom": 131072,
"beta-fast": 32,
"beta-slow": 1,
"rope-scaling-factor": 4,
"rope-scaling-mscale": 1.0,
"rope-scaling-mscale-all-dim": 0.0,
"rope-scaling-original-max-position-embeddings": 32768,
"rope-scaling-type": "yarn",
"use-flash-attn": null,
"bf16": null,
"transformer-impl": "local"
}
}
],
"test_qwen2_mcore_mmlu_evaluate": [
{
"param": {
"use-mcore-models": null,
"use-kv-cache": null,
"task-data-path": "/data/ci/datasets/eval_data/mmlu/data/test",
"task": "mmlu",
"tensor-model-parallel-size": 1,
"pipeline-model-parallel-size": 1,
"micro-batch-size": 2,
"seq-length": 4096,
"max-position-embeddings": 4096,
"tokenizer-type": "PretrainedFromHF",
"tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
"max-new-tokens": 1,
"make-vocab-size-divisible-by": 1,
"padded-vocab-size": 151936,
"rotary-base": 1000000,
"num-layers": 1,
"hidden-size": 1536,
"ffn-hidden-size": 8960,
"num-attention-heads": 12,
"group-query-attention": null,
"num-query-groups": 2,
"add-qkv-bias": null,
"disable-bias-linear": null,
"swiglu": null,
"position-embedding-type": "rope",
"load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
"normalization": "RMSNorm",
"norm-epsilon": 1e-06,
"tokenizer-not-use-fast": null,
"exit-on-missing-checkpoint": null,
"no-load-rng": null,
"no-load-optim": null,
"no-gradient-accumulation-fusion": null,
"attention-softmax-in-fp32": null,
"seed": 42,
"bf16": null,
"no-chat-template": null,
"transformer-impl": "local"
}
}
],
"test_cmmlu_evaluate": [
{
"param": {
"use-mcore-models": null,
"use-kv-cache": null,
"task-data-path": "/data/ci/datasets/eval_data/cmmlu/one_test",
"task": "cmmlu",
"tensor-model-parallel-size": 1,
"pipeline-model-parallel-size": 1,
"micro-batch-size": 2,
"seq-length": 4096,
"max-position-embeddings": 4096,
"tokenizer-type": "PretrainedFromHF",
"tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
"max-new-tokens": 1,
"make-vocab-size-divisible-by": 1,
"padded-vocab-size": 151936,
"rotary-base": 1000000,
"num-layers": 1,
"hidden-size": 1536,
"ffn-hidden-size": 8960,
"num-attention-heads": 12,
"group-query-attention": null,
"num-query-groups": 2,
"add-qkv-bias": null,
"disable-bias-linear": null,
"swiglu": null,
"position-embedding-type": "rope",
"load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
"normalization": "RMSNorm",
"norm-epsilon": 1e-06,
"tokenizer-not-use-fast": null,
"exit-on-missing-checkpoint": null,
"no-load-rng": null,
"no-load-optim": null,
"no-gradient-accumulation-fusion": null,
"attention-softmax-in-fp32": null,
"seed": 42,
"bf16": null,
"no-chat-template": null,
"transformer-impl": "local"
}
}
],
"test_humaneval_evaluate": [
{
"param": {
"use-mcore-models": null,
"use-kv-cache": null,
"task-data-path": "/data/ci/datasets/eval_data/human_eval/",
"task": "human_eval",
"tensor-model-parallel-size": 1,
"pipeline-model-parallel-size": 1,
"micro-batch-size": 2,
"seq-length": 4096,
"max-position-embeddings": 4096,
"tokenizer-type": "PretrainedFromHF",
"tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
"max-new-tokens": 1,
"make-vocab-size-divisible-by": 1,
"padded-vocab-size": 151936,
"rotary-base": 1000000,
"num-layers": 1,
"hidden-size": 1536,
"ffn-hidden-size": 8960,
"num-attention-heads": 12,
"group-query-attention": null,
"num-query-groups": 2,
"add-qkv-bias": null,
"disable-bias-linear": null,
"swiglu": null,
"position-embedding-type": "rope",
"load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
"normalization": "RMSNorm",
"norm-epsilon": 1e-06,
"tokenizer-not-use-fast": null,
"exit-on-missing-checkpoint": null,
"no-load-rng": null,
"no-load-optim": null,
"no-gradient-accumulation-fusion": null,
"attention-softmax-in-fp32": null,
"seed": 42,
"bf16": null,
"no-chat-template": null,
"transformer-impl": "local"
}
}
],
"test_ceval_evaluate": [
{
"param": {
"use-mcore-models": null,
"use-kv-cache": null,
"task-data-path": "/data/ci/datasets/eval_data/ceval/one_test/",
"task": "ceval",
"tensor-model-parallel-size": 1,
"pipeline-model-parallel-size": 1,
"micro-batch-size": 2,
"seq-length": 4096,
"max-position-embeddings": 4096,
"tokenizer-type": "PretrainedFromHF",
"tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
"max-new-tokens": 1,
"make-vocab-size-divisible-by": 1,
"padded-vocab-size": 151936,
"rotary-base": 1000000,
"num-layers": 1,
"hidden-size": 1536,
"ffn-hidden-size": 8960,
"num-attention-heads": 12,
"group-query-attention": null,
"num-query-groups": 2,
"add-qkv-bias": null,
"disable-bias-linear": null,
"swiglu": null,
"position-embedding-type": "rope",
"load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
"normalization": "RMSNorm",
"norm-epsilon": 1e-06,
"tokenizer-not-use-fast": null,
"exit-on-missing-checkpoint": null,
"no-load-rng": null,
"no-load-optim": null,
"no-gradient-accumulation-fusion": null,
"attention-softmax-in-fp32": null,
"seed": 42,
"bf16": null,
"no-chat-template": null,
"transformer-impl": "local"
}
}
],
"test_boolq_evaluate": [
{
"param": {
"use-mcore-models": null,
"use-kv-cache": null,
"task-data-path": "/data/ci/datasets/eval_data/boolq/dev/",
"task": "boolq",
"tensor-model-parallel-size": 1,
"pipeline-model-parallel-size": 1,
"micro-batch-size": 2,
"seq-length": 4096,
"max-position-embeddings": 4096,
"tokenizer-type": "PretrainedFromHF",
"tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
"max-new-tokens": 1,
"make-vocab-size-divisible-by": 1,
"padded-vocab-size": 151936,
"rotary-base": 1000000,
"num-layers": 1,
"hidden-size": 1536,
"ffn-hidden-size": 8960,
"num-attention-heads": 12,
"group-query-attention": null,
"num-query-groups": 2,
"add-qkv-bias": null,
"disable-bias-linear": null,
"swiglu": null,
"position-embedding-type": "rope",
"load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
"normalization": "RMSNorm",
"norm-epsilon": 1e-06,
"tokenizer-not-use-fast": null,
"exit-on-missing-checkpoint": null,
"no-load-rng": null,
"no-load-optim": null,
"no-gradient-accumulation-fusion": null,
"attention-softmax-in-fp32": null,
"seed": 42,
"bf16": null,
"no-chat-template": null,
"transformer-impl": "local"
}
}
],
"test_gsm8k_evaluate": [
{
"param": {
"use-mcore-models": null,
"use-kv-cache": null,
"task-data-path": "/data/ci/datasets/eval_data/gsm8k/one_test",
"task": "gsm8k",
"tensor-model-parallel-size": 1,
"pipeline-model-parallel-size": 1,
"micro-batch-size": 2,
"seq-length": 4096,
"max-position-embeddings": 4096,
"tokenizer-type": "PretrainedFromHF",
"tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
"max-new-tokens": 1,
"make-vocab-size-divisible-by": 1,
"padded-vocab-size": 151936,
"rotary-base": 1000000,
"num-layers": 1,
"hidden-size": 1536,
"ffn-hidden-size": 8960,
"num-attention-heads": 12,
"group-query-attention": null,
"num-query-groups": 2,
"add-qkv-bias": null,
"disable-bias-linear": null,
"swiglu": null,
"position-embedding-type": "rope",
"load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
"normalization": "RMSNorm",
"norm-epsilon": 1e-06,
"tokenizer-not-use-fast": null,
"exit-on-missing-checkpoint": null,
"no-load-rng": null,
"no-load-optim": null,
"no-gradient-accumulation-fusion": null,
"attention-softmax-in-fp32": null,
"seed": 42,
"bf16": null,
"no-chat-template": null,
"transformer-impl": "local"
}
}
]
}
