Star217
341
代码
代码
Issues23
Pull Requests99
Actions
项目讨论
Wiki
分析
Star217
341
已合并
[pytorch][refact]weight path fix #3599
guihaowen666创建于 2025年10月30日
[pytorch][refact]weight path fix #3599
已合并
guihaowen666创建于 2025年10月30日
从已删除 :br_220_weight_path_fix合入到Ascend/MindSpeed-LLM2.2.0
共 64 个文件变更+354-354
Mtests/coverage/checkpoint/test_checkpoint.json+15-15
@@ -9,8 +9,8 @@
9 "target-pipeline-parallel-size": "4",9 "target-pipeline-parallel-size": "4",
10 "target-expert-parallel-size": "8",10 "target-expert-parallel-size": "8",
11 "num-layers-per-virtual-pipeline-stage": "1",11 "num-layers-per-virtual-pipeline-stage": "1",
12- "load-dir":"/data/hf/deepseek2_hf",12+ "load-dir":"/data/ci/models/deepseek-v2/hf/deepseek2_hf",
13- "save-dir":"/data/ci/deepseek2/mg/deepseek2_mg",13+ "save-dir":"/data/ci/cache/deepseek2_tp1pp4ep8",
14 "use-mcore-models": null,14 "use-mcore-models": null,
15 "moe-grouped-gemm": null,15 "moe-grouped-gemm": null,
16 "model-type-hf": "deepseek2",16 "model-type-hf": "deepseek2",
@@ -66,8 +66,8 @@
66 "target-tensor-parallel-size": "1",66 "target-tensor-parallel-size": "1",
67 "target-pipeline-parallel-size": "1",67 "target-pipeline-parallel-size": "1",
68 "target-expert-parallel-size": "1",68 "target-expert-parallel-size": "1",
69- "save-dir":"/data/hf/deepseek2_hf",69+ "save-dir":"/data/ci/models/deepseek-v2/hf/deepseek2_hf",
70- "load-dir":"/data/ci/deepseek2/mg/deepseek2_mg",70+ "load-dir":"/data/ci/cache/deepseek2_tp1pp4ep8",
71 "use-mcore-models": null,71 "use-mcore-models": null,
72 "moe-grouped-gemm": null,72 "moe-grouped-gemm": null,
73 "model-type-hf": "deepseek2",73 "model-type-hf": "deepseek2",
@@ -91,8 +91,8 @@
91 "save-model-type":"mg",91 "save-model-type":"mg",
92 "target-tensor-parallel-size": "4",92 "target-tensor-parallel-size": "4",
93 "target-pipeline-parallel-size": "2",93 "target-pipeline-parallel-size": "2",
94- "load-dir":"/data/hf/Qwen2.5-7B",94+ "load-dir":"/data/ci/models/qwen25/hf/Qwen2.5-7B",
95- "save-dir":"/data/ci/Qwen2.5-mg",95+ "save-dir":"/data/ci/cache/Qwen2.5-mg",
96 "tokenizer-model":"/data/ci/Qwen2.5-7B",96 "tokenizer-model":"/data/ci/Qwen2.5-7B",
97 "use-mcore-models": null,97 "use-mcore-models": null,
98 "add-qkv-bias": null,98 "add-qkv-bias": null,
@@ -123,8 +123,8 @@
123 "save-model-type":"hf",123 "save-model-type":"hf",
124 "target-tensor-parallel-size": "1",124 "target-tensor-parallel-size": "1",
125 "target-pipeline-parallel-size": "1",125 "target-pipeline-parallel-size": "1",
126- "load-dir":"/data/ci/Qwen2.5-mg",126+ "load-dir":"/data/ci/cache/Qwen2.5-mg",
127- "save-dir":"/data/hf/Qwen2.5-7B",127+ "save-dir":"/data/ci/models/qwen25/hf/Qwen2.5-7B",
128 "use-mcore-models": null,128 "use-mcore-models": null,
129 "add-qkv-bias": null,129 "add-qkv-bias": null,
130 "model-type-hf": "llama2",130 "model-type-hf": "llama2",
@@ -149,12 +149,12 @@
149 "save-model-type": "mg",149 "save-model-type": "mg",
150 "target-tensor-parallel-size": "2",150 "target-tensor-parallel-size": "2",
151 "target-pipeline-parallel-size": "2",151 "target-pipeline-parallel-size": "2",
152- "save-dir":"/data/llama-3-8b-hf-nooplayer-tp2pp2vpp2-mcore-test/",152+ "save-dir":"/data/ci/cache/llama-3-8b-hf-nooplayer-tp2pp2vpp2-mcore-test/",
153- "load-dir":"/data/hf/llama-3-8b-hf-layer14/",153+ "load-dir":"/data/ci/models/llama3/hf/llama-3-8b-hf-layer14/",
154 "num-layers-per-virtual-pipeline-stage": "2",154 "num-layers-per-virtual-pipeline-stage": "2",
155 "noop-layers": "1,15",155 "noop-layers": "1,15",
156 "params-dtype": "bf16",156 "params-dtype": "bf16",
157- "tokenizer-model": "/data/llama-3-8b-hf/tokenizer.json"157+ "tokenizer-model": "/data/ci/models/llama3/hf/llama-3-8b-hf/tokenizer.json"
158 }158 }
159 },159 },
160 {160 {
@@ -177,9 +177,9 @@
177 "save-model-type": "mg",177 "save-model-type": "mg",
178 "target-tensor-parallel-size": "2",178 "target-tensor-parallel-size": "2",
179 "target-pipeline-parallel-size": "4",179 "target-pipeline-parallel-size": "4",
180- "save-dir":"/data/llama2_merge_lora_ckpt/merge_lora_mcore/",180+ "save-dir":"/data/ci/cache/llama2_merge_lora_mcore/",
181- "load-dir":"/data/pipeline/llama2_merge_lora_ckpt/Llama2_mcore_l4tp4pp2/",181+ "load-dir":"/data/ci/models/llama2/mg/Llama2_mcore_l4tp4pp2/",
182- "lora-load":"/data/pipeline/llama2_merge_lora_ckpt/llama2_l4_tp4pp2_lora/",182+ "lora-load":"/data/ci/models/llama2/mg/llama2_l4_tp4pp2_lora/",
183 "lora-r":"16",183 "lora-r":"16",
184 "lora-alpha":"32",184 "lora-alpha":"32",
185 "lora-target-modules":"linear_qkv linear_proj linear_fc1 linear_fc2"185 "lora-target-modules":"linear_qkv linear_proj linear_fc1 linear_fc2"
@@ -212,7 +212,7 @@
212 "target-pipeline-parallel-size": "1",212 "target-pipeline-parallel-size": "1",
213 "target-expert-parallel-size": "1",213 "target-expert-parallel-size": "1",
214 "save-dir":"/data/hf/Mixtral-mcore-hf/",214 "save-dir":"/data/hf/Mixtral-mcore-hf/",
215- "load-dir":"/data/pipeline/mixtral_lora_filter/",215+ "load-dir":"/data/ci/models/mixtral/mg/mixtral_lora_filter/",
216 "lora-r":"8",216 "lora-r":"8",
217 "lora-alpha":"16",217 "lora-alpha":"16",
218 "lora-target-modules":"linear_qkv linear_proj linear_fc1 linear_fc2",218 "lora-target-modules":"linear_qkv linear_proj linear_fc1 linear_fc2",
Mtests/coverage/evaluation/test_evaluate.json+27-27
@@ -2,7 +2,7 @@
2 "test_llama2_mcore_agieval_evaluate": [2 "test_llama2_mcore_agieval_evaluate": [
3 {3 {
4 "param": {4 "param": {
5- "task-data-path":"/data/eval_data/agieval",5+ "task-data-path":"/data/ci/datasets/eval_data/agieval",
6 "task":"agieval",6 "task":"agieval",
7 "tensor-model-parallel-size": 8,7 "tensor-model-parallel-size": 8,
8 "pipeline-model-parallel-size": 1,8 "pipeline-model-parallel-size": 1,
@@ -27,9 +27,9 @@
27 "fp16": null,27 "fp16": null,
28 "no-load-rng": null,28 "no-load-rng": null,
29 "no-load-optim": null,29 "no-load-optim": null,
30- "load":"/data/ci/ckpt",30+ "load":"/data/ci/models/llama2/mg/llama2-tp8pp1",
31 "tokenizer-type":"PretrainedFromHF",31 "tokenizer-type":"PretrainedFromHF",
32- "tokenizer-name-or-path":"/data/hf/llama-2-7b-hf",32+ "tokenizer-name-or-path":"/data/ci/models/llama2/hf/llama-2-7b-hf",
33 "npu-deterministic": null33 "npu-deterministic": null
34 }34 }
35 }35 }
@@ -38,7 +38,7 @@
38 "test_llama2_mcore_bbh_evaluate": [38 "test_llama2_mcore_bbh_evaluate": [
39 {39 {
40 "param": {40 "param": {
41- "task-data-path":"/data/eval_data/bbh/test",41+ "task-data-path":"/data/ci/datasets/eval_data/bbh/test",
42 "task":"bbh",42 "task":"bbh",
43 "tensor-model-parallel-size": 8,43 "tensor-model-parallel-size": 8,
44 "pipeline-model-parallel-size": 1,44 "pipeline-model-parallel-size": 1,
@@ -63,9 +63,9 @@
63 "fp16": null,63 "fp16": null,
64 "no-load-rng": null,64 "no-load-rng": null,
65 "no-load-optim": null,65 "no-load-optim": null,
66- "load":"/data/ci/ckpt",66+ "load":"/data/ci/models/llama2/mg/llama2-tp8pp1",
67 "tokenizer-type":"PretrainedFromHF",67 "tokenizer-type":"PretrainedFromHF",
68- "tokenizer-name-or-path":"/data/hf/llama-2-7b-hf",68+ "tokenizer-name-or-path":"/data/ci/models/llama2/hf/llama-2-7b-hf",
69 "npu-deterministic": null69 "npu-deterministic": null
70 }70 }
71 }71 }
@@ -75,7 +75,7 @@
75 {75 {
76 "param": {76 "param": {
77 "use-mcore-models": null,77 "use-mcore-models": null,
78- "task-data-path": "/data/eval_data/NeedleBench/",78+ "task-data-path": "/data/ci/datasets/eval_data/NeedleBench/",
79 "task": "needlebench",79 "task": "needlebench",
80 "tensor-model-parallel-size": 4,80 "tensor-model-parallel-size": 4,
81 "pipeline-model-parallel-size": 1,81 "pipeline-model-parallel-size": 1,
@@ -89,10 +89,10 @@
89 "disable-bias-linear": null,89 "disable-bias-linear": null,
90 "swiglu": null,90 "swiglu": null,
91 "position-embedding-type": "rope",91 "position-embedding-type": "rope",
92- "load": "/data/qwen2-7b-2layer-tp4pp1/",92+ "load": "/data/ci/models/qwen2-7b/mg/qwen2-7b-2layer-tp4pp1/",
93 "normalization": "RMSNorm",93 "normalization": "RMSNorm",
94 "tokenizer-type": "PretrainedFromHF",94 "tokenizer-type": "PretrainedFromHF",
95- "tokenizer-name-or-path": "/data/qwen2-7b-hf/",95+ "tokenizer-name-or-path": "/data/ci/models/qwen2-7b/hf/qwen2-7b-hf/",
96 "tokenizer-not-use-fast": null,96 "tokenizer-not-use-fast": null,
97 "micro-batch-size": 1,97 "micro-batch-size": 1,
98 "use-kv-cache": null,98 "use-kv-cache": null,
@@ -128,7 +128,7 @@
128 "param": {128 "param": {
129 "use-mcore-models": null,129 "use-mcore-models": null,
130 "use-kv-cache": null,130 "use-kv-cache": null,
131- "task-data-path": "/data/eval_data/mmlu/data/test",131+ "task-data-path": "/data/ci/datasets/eval_data/mmlu/data/test",
132 "task": "mmlu",132 "task": "mmlu",
133 "tensor-model-parallel-size": 1,133 "tensor-model-parallel-size": 1,
134 "pipeline-model-parallel-size": 1,134 "pipeline-model-parallel-size": 1,
@@ -136,7 +136,7 @@
136 "seq-length": 4096,136 "seq-length": 4096,
137 "max-position-embeddings": 4096,137 "max-position-embeddings": 4096,
138 "tokenizer-type": "PretrainedFromHF",138 "tokenizer-type": "PretrainedFromHF",
139- "tokenizer-name-or-path": "/data/Qwen2-1.5B",139+ "tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
140 "max-new-tokens": 1,140 "max-new-tokens": 1,
141 "make-vocab-size-divisible-by": 1,141 "make-vocab-size-divisible-by": 1,
142 "padded-vocab-size": 151936,142 "padded-vocab-size": 151936,
@@ -151,7 +151,7 @@
151 "disable-bias-linear": null,151 "disable-bias-linear": null,
152 "swiglu": null,152 "swiglu": null,
153 "position-embedding-type": "rope",153 "position-embedding-type": "rope",
154- "load": "/data/qwen2-1.5b-1layer-tp1pp1",154+ "load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
155 "normalization": "RMSNorm",155 "normalization": "RMSNorm",
156 "norm-epsilon": 1e-06,156 "norm-epsilon": 1e-06,
157 "tokenizer-not-use-fast": null,157 "tokenizer-not-use-fast": null,
@@ -172,7 +172,7 @@
172 "param": {172 "param": {
173 "use-mcore-models": null,173 "use-mcore-models": null,
174 "use-kv-cache": null,174 "use-kv-cache": null,
175- "task-data-path": "/data/eval_data/cmmlu/one_test",175+ "task-data-path": "/data/ci/datasets/eval_data/cmmlu/one_test",
176 "task": "cmmlu",176 "task": "cmmlu",
177 "tensor-model-parallel-size": 1,177 "tensor-model-parallel-size": 1,
178 "pipeline-model-parallel-size": 1,178 "pipeline-model-parallel-size": 1,
@@ -180,7 +180,7 @@
180 "seq-length": 4096,180 "seq-length": 4096,
181 "max-position-embeddings": 4096,181 "max-position-embeddings": 4096,
182 "tokenizer-type": "PretrainedFromHF",182 "tokenizer-type": "PretrainedFromHF",
183- "tokenizer-name-or-path": "/data/Qwen2-1.5B",183+ "tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
184 "max-new-tokens": 1,184 "max-new-tokens": 1,
185 "make-vocab-size-divisible-by": 1,185 "make-vocab-size-divisible-by": 1,
186 "padded-vocab-size": 151936,186 "padded-vocab-size": 151936,
@@ -195,7 +195,7 @@
195 "disable-bias-linear": null,195 "disable-bias-linear": null,
196 "swiglu": null,196 "swiglu": null,
197 "position-embedding-type": "rope",197 "position-embedding-type": "rope",
198- "load": "/data/qwen2-1.5b-1layer-tp1pp1",198+ "load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
199 "normalization": "RMSNorm",199 "normalization": "RMSNorm",
200 "norm-epsilon": 1e-06,200 "norm-epsilon": 1e-06,
201 "tokenizer-not-use-fast": null,201 "tokenizer-not-use-fast": null,
@@ -216,7 +216,7 @@
216 "param": {216 "param": {
217 "use-mcore-models": null,217 "use-mcore-models": null,
218 "use-kv-cache": null,218 "use-kv-cache": null,
219- "task-data-path": "/data/eval_data/human_eval/",219+ "task-data-path": "/data/ci/datasets/eval_data/human_eval/",
220 "task": "human_eval",220 "task": "human_eval",
221 "tensor-model-parallel-size": 1,221 "tensor-model-parallel-size": 1,
222 "pipeline-model-parallel-size": 1,222 "pipeline-model-parallel-size": 1,
@@ -224,7 +224,7 @@
224 "seq-length": 4096,224 "seq-length": 4096,
225 "max-position-embeddings": 4096,225 "max-position-embeddings": 4096,
226 "tokenizer-type": "PretrainedFromHF",226 "tokenizer-type": "PretrainedFromHF",
227- "tokenizer-name-or-path": "/data/Qwen2-1.5B",227+ "tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
228 "max-new-tokens": 1,228 "max-new-tokens": 1,
229 "make-vocab-size-divisible-by": 1,229 "make-vocab-size-divisible-by": 1,
230 "padded-vocab-size": 151936,230 "padded-vocab-size": 151936,
@@ -239,7 +239,7 @@
239 "disable-bias-linear": null,239 "disable-bias-linear": null,
240 "swiglu": null,240 "swiglu": null,
241 "position-embedding-type": "rope",241 "position-embedding-type": "rope",
242- "load": "/data/qwen2-1.5b-1layer-tp1pp1",242+ "load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
243 "normalization": "RMSNorm",243 "normalization": "RMSNorm",
244 "norm-epsilon": 1e-06,244 "norm-epsilon": 1e-06,
245 "tokenizer-not-use-fast": null,245 "tokenizer-not-use-fast": null,
@@ -260,7 +260,7 @@
260 "param": {260 "param": {
261 "use-mcore-models": null,261 "use-mcore-models": null,
262 "use-kv-cache": null,262 "use-kv-cache": null,
263- "task-data-path": "/data/eval_data/ceval/one_test/",263+ "task-data-path": "/data/ci/datasets/eval_data/ceval/one_test/",
264 "task": "ceval",264 "task": "ceval",
265 "tensor-model-parallel-size": 1,265 "tensor-model-parallel-size": 1,
266 "pipeline-model-parallel-size": 1,266 "pipeline-model-parallel-size": 1,
@@ -268,7 +268,7 @@
268 "seq-length": 4096,268 "seq-length": 4096,
269 "max-position-embeddings": 4096,269 "max-position-embeddings": 4096,
270 "tokenizer-type": "PretrainedFromHF",270 "tokenizer-type": "PretrainedFromHF",
271- "tokenizer-name-or-path": "/data/Qwen2-1.5B",271+ "tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
272 "max-new-tokens": 1,272 "max-new-tokens": 1,
273 "make-vocab-size-divisible-by": 1,273 "make-vocab-size-divisible-by": 1,
274 "padded-vocab-size": 151936,274 "padded-vocab-size": 151936,
@@ -283,7 +283,7 @@
283 "disable-bias-linear": null,283 "disable-bias-linear": null,
284 "swiglu": null,284 "swiglu": null,
285 "position-embedding-type": "rope",285 "position-embedding-type": "rope",
286- "load": "/data/qwen2-1.5b-1layer-tp1pp1",286+ "load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
287 "normalization": "RMSNorm",287 "normalization": "RMSNorm",
288 "norm-epsilon": 1e-06,288 "norm-epsilon": 1e-06,
289 "tokenizer-not-use-fast": null,289 "tokenizer-not-use-fast": null,
@@ -304,7 +304,7 @@
304 "param": {304 "param": {
305 "use-mcore-models": null,305 "use-mcore-models": null,
306 "use-kv-cache": null,306 "use-kv-cache": null,
307- "task-data-path": "/data/eval_data/boolq/dev/",307+ "task-data-path": "/data/ci/datasets/eval_data/boolq/dev/",
308 "task": "boolq",308 "task": "boolq",
309 "tensor-model-parallel-size": 1,309 "tensor-model-parallel-size": 1,
310 "pipeline-model-parallel-size": 1,310 "pipeline-model-parallel-size": 1,
@@ -312,7 +312,7 @@
312 "seq-length": 4096,312 "seq-length": 4096,
313 "max-position-embeddings": 4096,313 "max-position-embeddings": 4096,
314 "tokenizer-type": "PretrainedFromHF",314 "tokenizer-type": "PretrainedFromHF",
315- "tokenizer-name-or-path": "/data/Qwen2-1.5B",315+ "tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
316 "max-new-tokens": 1,316 "max-new-tokens": 1,
317 "make-vocab-size-divisible-by": 1,317 "make-vocab-size-divisible-by": 1,
318 "padded-vocab-size": 151936,318 "padded-vocab-size": 151936,
@@ -327,7 +327,7 @@
327 "disable-bias-linear": null,327 "disable-bias-linear": null,
328 "swiglu": null,328 "swiglu": null,
329 "position-embedding-type": "rope",329 "position-embedding-type": "rope",
330- "load": "/data/qwen2-1.5b-1layer-tp1pp1",330+ "load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
331 "normalization": "RMSNorm",331 "normalization": "RMSNorm",
332 "norm-epsilon": 1e-06,332 "norm-epsilon": 1e-06,
333 "tokenizer-not-use-fast": null,333 "tokenizer-not-use-fast": null,
@@ -348,7 +348,7 @@
348 "param": {348 "param": {
349 "use-mcore-models": null,349 "use-mcore-models": null,
350 "use-kv-cache": null,350 "use-kv-cache": null,
351- "task-data-path": "/data/eval_data/gsm8k/one_test",351+ "task-data-path": "/data/ci/datasets/eval_data/gsm8k/one_test",
352 "task": "gsm8k",352 "task": "gsm8k",
353 "tensor-model-parallel-size": 1,353 "tensor-model-parallel-size": 1,
354 "pipeline-model-parallel-size": 1,354 "pipeline-model-parallel-size": 1,
@@ -356,7 +356,7 @@
356 "seq-length": 4096,356 "seq-length": 4096,
357 "max-position-embeddings": 4096,357 "max-position-embeddings": 4096,
358 "tokenizer-type": "PretrainedFromHF",358 "tokenizer-type": "PretrainedFromHF",
359- "tokenizer-name-or-path": "/data/Qwen2-1.5B",359+ "tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
360 "max-new-tokens": 1,360 "max-new-tokens": 1,
361 "make-vocab-size-divisible-by": 1,361 "make-vocab-size-divisible-by": 1,
362 "padded-vocab-size": 151936,362 "padded-vocab-size": 151936,
@@ -371,7 +371,7 @@
371 "disable-bias-linear": null,371 "disable-bias-linear": null,
372 "swiglu": null,372 "swiglu": null,
373 "position-embedding-type": "rope",373 "position-embedding-type": "rope",
374- "load": "/data/qwen2-1.5b-1layer-tp1pp1",374+ "load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-1layer-tp1pp1",
375 "normalization": "RMSNorm",375 "normalization": "RMSNorm",
376 "norm-epsilon": 1e-06,376 "norm-epsilon": 1e-06,
377 "tokenizer-not-use-fast": null,377 "tokenizer-not-use-fast": null,
Mtests/coverage/inference/test_inference.json+17-17
@@ -23,10 +23,10 @@
23 "make-vocab-size-divisible-by": 1,23 "make-vocab-size-divisible-by": 1,
24 "swiglu": null,24 "swiglu": null,
25 "normalization": "RMSNorm", 25 "normalization": "RMSNorm",
26- "load":"/data/ci/ckpt",26+ "load":"/data/ci/models/llama2/mg/llama2-tp8pp1",
27 "tokenizer-type": "PretrainedFromHF", 27 "tokenizer-type": "PretrainedFromHF",
28- "tokenizer-name-or-path":"/data/hf/llama-2-7b-hf",28+ "tokenizer-name-or-path":"/data/ci/models/llama2/hf/llama-2-7b-hf",
29- "tokenizer-model": "/data/hf/llama-2-7b-hf/tokenizer.model",29+ "tokenizer-model": "/data/ci/models/llama2/hf/llama-2-7b-hf/tokenizer.model",
30 "disable-bias-linear": null,30 "disable-bias-linear": null,
31 "attention-softmax-in-fp32": null,31 "attention-softmax-in-fp32": null,
32 "untie-embeddings-and-output-weights": null,32 "untie-embeddings-and-output-weights": null,
@@ -65,10 +65,10 @@
65 "make-vocab-size-divisible-by": 1,65 "make-vocab-size-divisible-by": 1,
66 "swiglu": null,66 "swiglu": null,
67 "normalization": "RMSNorm",67 "normalization": "RMSNorm",
68- "load":"/data/ci/ckpt",68+ "load":"/data/ci/models/llama2/mg/llama2-tp8pp1",
69 "tokenizer-type": "PretrainedFromHF",69 "tokenizer-type": "PretrainedFromHF",
70- "tokenizer-name-or-path":"/data/hf/llama-2-7b-hf",70+ "tokenizer-name-or-path":"/data/ci/models/llama2/hf/llama-2-7b-hf",
71- "tokenizer-model": "/data/hf/llama-2-7b-hf/tokenizer.model",71+ "tokenizer-model": "/data/ci/models/llama2/hf/llama-2-7b-hf/tokenizer.model",
72 "disable-bias-linear": null,72 "disable-bias-linear": null,
73 "attention-softmax-in-fp32": null,73 "attention-softmax-in-fp32": null,
74 "untie-embeddings-and-output-weights": null,74 "untie-embeddings-and-output-weights": null,
@@ -103,7 +103,7 @@
103 "max-position-embeddings": 163840,103 "max-position-embeddings": 163840,
104 "position-embedding-type": "rope",104 "position-embedding-type": "rope",
105 "swiglu": null,105 "swiglu": null,
106- "load": "/data/deepseek2-l4-tp1-pp1-ep8",106+ "load": "/data/ci/models/deepseek-v2/mg/deepseek2-l4-tp1-pp1-ep8",
107 "tokenizer-type": "PretrainedFromHF",107 "tokenizer-type": "PretrainedFromHF",
108 "tokenizer-name-or-path": "/data/gemm/",108 "tokenizer-name-or-path": "/data/gemm/",
109 "bf16": null,109 "bf16": null,
@@ -183,7 +183,7 @@
183 "max-position-embeddings": 163840,183 "max-position-embeddings": 163840,
184 "position-embedding-type": "rope",184 "position-embedding-type": "rope",
185 "swiglu": null,185 "swiglu": null,
186- "load": "/data/deepseek2-l4-tp1-pp1-ep8",186+ "load": "/data/ci/models/deepseek-v2/mg/deepseek2-l4-tp1-pp1-ep8",
187 "tokenizer-type": "PretrainedFromHF",187 "tokenizer-type": "PretrainedFromHF",
188 "tokenizer-name-or-path": "/data/gemm/",188 "tokenizer-name-or-path": "/data/gemm/",
189 "bf16": null,189 "bf16": null,
@@ -272,10 +272,10 @@
272 "norm-epsilon": 1e-5,272 "norm-epsilon": 1e-5,
273 "hidden-dropout": 0,273 "hidden-dropout": 0,
274 "attention-dropout": 0,274 "attention-dropout": 0,
275- "load": "/data/llama3-8b-mcore-tp2-pp4",275+ "load": "/data/ci/models/llama3/mg/llama3-8b-mcore-tp2-pp4",
276 "tokenizer-type": "PretrainedFromHF", 276 "tokenizer-type": "PretrainedFromHF",
277- "tokenizer-name-or-path":"/data/llama-3-8b-hf",277+ "tokenizer-name-or-path":"/data/ci/models/llama3/hf/llama-3-8b-hf",
278- "tokenizer-model": "/data/llama-3-8b-hf/tokenizer.model", 278+ "tokenizer-model": "/data/ci/models/llama3/hf/llama-3-8b-hf/tokenizer.model",
279 "disable-bias-linear": null,279 "disable-bias-linear": null,
280 "attention-softmax-in-fp32": null, 280 "attention-softmax-in-fp32": null,
281 "untie-embeddings-and-output-weights": null, 281 "untie-embeddings-and-output-weights": null,
@@ -318,10 +318,10 @@
318 "norm-epsilon": 1e-5,318 "norm-epsilon": 1e-5,
319 "hidden-dropout": 0,319 "hidden-dropout": 0,
320 "attention-dropout": 0,320 "attention-dropout": 0,
321- "load": "/data/llama3-8b-mcore-tp2-pp4",321+ "load": "/data/ci/models/llama3/mg/llama3-8b-mcore-tp2-pp4",
322 "tokenizer-type": "PretrainedFromHF",322 "tokenizer-type": "PretrainedFromHF",
323- "tokenizer-name-or-path":"/data/llama-3-8b-hf",323+ "tokenizer-name-or-path":"/data/ci/models/llama3/hf/llama-3-8b-hf",
324- "tokenizer-model": "/data/llama-3-8b-hf/tokenizer.model",324+ "tokenizer-model": "/data/ci/models/llama3/hf/llama-3-8b-hf/tokenizer.model",
325 "disable-bias-linear": null,325 "disable-bias-linear": null,
326 "attention-softmax-in-fp32": null,326 "attention-softmax-in-fp32": null,
327 "untie-embeddings-and-output-weights": null,327 "untie-embeddings-and-output-weights": null,
@@ -363,10 +363,10 @@
363 "norm-epsilon": 1e-5,363 "norm-epsilon": 1e-5,
364 "hidden-dropout": 0,364 "hidden-dropout": 0,
365 "attention-dropout": 0,365 "attention-dropout": 0,
366- "load": "/data/llama3-8b-mcore-tp2-pp4",366+ "load": "/data/ci/models/llama3/mg/llama3-8b-mcore-tp2-pp4",
367 "tokenizer-type": "PretrainedFromHF",367 "tokenizer-type": "PretrainedFromHF",
368- "tokenizer-name-or-path":"/data/llama-3-8b-hf",368+ "tokenizer-name-or-path":"/data/ci/models/llama3/hf/llama-3-8b-hf",
369- "tokenizer-model": "/data/llama-3-8b-hf/tokenizer.model",369+ "tokenizer-model": "/data/ci/models/llama3/hf/llama-3-8b-hf/tokenizer.model",
370 "disable-bias-linear": null,370 "disable-bias-linear": null,
371 "attention-softmax-in-fp32": null,371 "attention-softmax-in-fp32": null,
372 "untie-embeddings-and-output-weights": null,372 "untie-embeddings-and-output-weights": null,
Mtests/coverage/process_data/test_process_instruction_data.json+19-19
@@ -2,22 +2,22 @@
2 "instruction_dataset": [2 "instruction_dataset": [
3 {3 {
4 "params" : {4 "params" : {
5- "input-dataset": "/data/process_dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet",5+ "input-dataset": "/data/ci/datasets/origin/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
6- "test-out-part": "/data/process_dataset/test_ins_subs/",6+ "test-out-part": "/data/ci/cache/process_dataset/test_ins_subs/",
7- "base-out-part": "/data/process_dataset/base_ins_subs/",7+ "base-out-part": "/data/ci/datasets/processed/base_ins_subs/",
8- "test-out-merge": "/data/process_dataset/test_ins_merge/",8+ "test-out-merge": "/data/ci/cache/process_dataset/test_ins_merge/",
9- "base-out-merge": "/data/process_dataset/base_ins_merge/"9+ "base-out-merge": "/data/ci/datasets/processed/base_ins_merge/"
10 }10 }
11 }11 }
12 ],12 ],
13 "test_instruction_datasets_part1": [13 "test_instruction_datasets_part1": [
14 {14 {
15 "params": {15 "params": {
16- "input": "/data/process_dataset/0001-alpaca.parquet",16+ "input": "/data/ci/datasets/origin/0001-alpaca.parquet",
17 "tokenizer-type": "PretrainedFromHF",17 "tokenizer-type": "PretrainedFromHF",
18 "handler-name": "GeneralInstructionHandler",18 "handler-name": "GeneralInstructionHandler",
19- "output-prefix": "/data/process_dataset/test_ins_subs/part1", 19+ "output-prefix": "/data/ci/cache/process_dataset/test_ins_subs/part1",
20- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf",20+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf",
21 "workers": 4,21 "workers": 4,
22 "log-interval": 1000,22 "log-interval": 1000,
23 "append-eod": null23 "append-eod": null
@@ -27,11 +27,11 @@
27 "test_instruction_datasets_part2": [27 "test_instruction_datasets_part2": [
28 {28 {
29 "params": {29 "params": {
30- "input": "/data/process_dataset/0002-alpaca.parquet",30+ "input": "/data/ci/datasets/origin/0002-alpaca.parquet",
31 "tokenizer-type": "PretrainedFromHF",31 "tokenizer-type": "PretrainedFromHF",
32 "handler-name": "GeneralInstructionHandler",32 "handler-name": "GeneralInstructionHandler",
33- "output-prefix": "/data/process_dataset/test_ins_subs/part2",33+ "output-prefix": "/data/ci/cache/process_dataset/test_ins_subs/part2",
34- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf",34+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf",
35 "workers": 4,35 "workers": 4,
36 "log-interval": 1000,36 "log-interval": 1000,
37 "append-eod": null37 "append-eod": null
@@ -41,8 +41,8 @@
41 "test_merge_instrction_datasets": [41 "test_merge_instrction_datasets": [
42 {42 {
43 "params": {43 "params": {
44- "input": "/data/process_dataset/test_ins_subs/",44+ "input": "/data/ci/cache/process_dataset/test_ins_subs/",
45- "output-prefix": "/data/process_dataset/test_ins_merge/merge",45+ "output-prefix": "/data/ci/cache/process_dataset/test_ins_merge/merge",
46 "merge-group-keys": ["packed_attention_mask_document", "packed_input_ids_document", "packed_labels_document"]46 "merge-group-keys": ["packed_attention_mask_document", "packed_input_ids_document", "packed_labels_document"]
47 }47 }
48 }48 }
@@ -58,11 +58,11 @@
58 "alpaca_style_instruction_handler": [58 "alpaca_style_instruction_handler": [
59 {59 {
60 "params": {60 "params": {
61- "input": "/data/process_dataset/0001-alpaca.parquet",61+ "input": "/data/ci/datasets/origin/0001-alpaca.parquet",
62 "tokenizer-type": "PretrainedFromHF",62 "tokenizer-type": "PretrainedFromHF",
63 "handler-name": "AlpacaStyleInstructionHandler",63 "handler-name": "AlpacaStyleInstructionHandler",
64 "output-prefix": "/data/process_dataset/test_instruction_handler/alpaca_style",64 "output-prefix": "/data/process_dataset/test_instruction_handler/alpaca_style",
65- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf",65+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf",
66 "workers": 4,66 "workers": 4,
67 "log-interval": 1000,67 "log-interval": 1000,
68 "overwrite-cache": null,68 "overwrite-cache": null,
@@ -73,11 +73,11 @@
73 "alpaca_style_pack_instruction_handler": [73 "alpaca_style_pack_instruction_handler": [
74 {74 {
75 "params": {75 "params": {
76- "input": "/data/process_dataset/0001-alpaca.parquet",76+ "input": "/data/ci/datasets/origin/0001-alpaca.parquet",
77 "tokenizer-type": "PretrainedFromHF",77 "tokenizer-type": "PretrainedFromHF",
78 "handler-name": "AlpacaStyleInstructionHandler",78 "handler-name": "AlpacaStyleInstructionHandler",
79 "output-prefix": "/data/process_dataset/test_instruction_handler/alpaca_style_pack",79 "output-prefix": "/data/process_dataset/test_instruction_handler/alpaca_style_pack",
80- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf",80+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf",
81 "workers": 4,81 "workers": 4,
82 "log-interval": 1000,82 "log-interval": 1000,
83 "overwrite-cache": null,83 "overwrite-cache": null,
@@ -91,11 +91,11 @@
91 "sharegpt_style_instruction_handler": [91 "sharegpt_style_instruction_handler": [
92 {92 {
93 "params": {93 "params": {
94- "input": "/data/process_dataset/sharegpt_formatted_data-evol-gpt4.jsonl",94+ "input": "/data/ci/datasets/origin/sharegpt_formatted_data-evol-gpt4.jsonl",
95 "tokenizer-type": "PretrainedFromHF",95 "tokenizer-type": "PretrainedFromHF",
96 "handler-name": "SharegptStyleInstructionHandler",96 "handler-name": "SharegptStyleInstructionHandler",
97 "output-prefix": "/data/process_dataset/test_instruction_handler/sharegpt_style",97 "output-prefix": "/data/process_dataset/test_instruction_handler/sharegpt_style",
98- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf",98+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf",
99 "workers": 4,99 "workers": 4,
100 "log-interval": 1000,100 "log-interval": 1000,
101 "prompt-type" : "llama2"101 "prompt-type" : "llama2"
Mtests/coverage/process_data/test_process_pretrain_data.json+13-13
@@ -2,21 +2,21 @@
2 "pretrain_dataset": [2 "pretrain_dataset": [
3 {3 {
4 "params" : {4 "params" : {
5- "input-dataset": "/data/process_dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet",5+ "input-dataset": "/data/ci/datasets/origin/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
6- "test-out-part": "/data/process_dataset/test_merge_subs/",6+ "test-out-part": "/data/ci/cache/process_dataset/test_merge_subs/",
7- "base-out-part": "/data/process_dataset/base_merge_subs/",7+ "base-out-part": "/data/ci/datasets/processed/base_merge_subs/",
8- "test-out-merge": "/data/process_dataset/test_merge/",8+ "test-out-merge": "/data/ci/cache/process_dataset/test_merge/",
9- "base-out-merge": "/data/process_dataset/base_merge/"9+ "base-out-merge": "/data/ci/datasets/processed/base_merge/"
10 }10 }
11 }11 }
12 ],12 ],
13 "test_pretrain_datasets_part1": [13 "test_pretrain_datasets_part1": [
14 {14 {
15 "params": {15 "params": {
16- "input": "/data/process_dataset/0001-alpaca.parquet",16+ "input": "/data/ci/datasets/origin/0001-alpaca.parquet",
17 "tokenizer-type": "PretrainedFromHF",17 "tokenizer-type": "PretrainedFromHF",
18- "output-prefix": "/data/process_dataset/test_merge_subs/part1", 18+ "output-prefix": "/data/ci/cache/process_dataset/test_merge_subs/part1",
19- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf",19+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf",
20 "workers": 4,20 "workers": 4,
21 "log-interval": 100021 "log-interval": 1000
22 }22 }
@@ -25,10 +25,10 @@
25 "test_pretrain_datasets_part2": [25 "test_pretrain_datasets_part2": [
26 {26 {
27 "params": {27 "params": {
28- "input": "/data/process_dataset/0002-alpaca.parquet",28+ "input": "/data/ci/datasets/origin/0002-alpaca.parquet",
29 "tokenizer-type": "PretrainedFromHF",29 "tokenizer-type": "PretrainedFromHF",
30- "output-prefix": "/data/process_dataset/test_merge_subs/part2",30+ "output-prefix": "/data/ci/cache/process_dataset/test_merge_subs/part2",
31- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf",31+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf",
32 "workers": 4,32 "workers": 4,
33 "log-interval": 100033 "log-interval": 1000
34 }34 }
@@ -37,8 +37,8 @@
37 "test_merge_pretrain_datasets": [37 "test_merge_pretrain_datasets": [
38 {38 {
39 "params": {39 "params": {
40- "input": "/data/process_dataset/test_merge_subs/",40+ "input": "/data/ci/cache/process_dataset/test_merge_subs/",
41- "output-prefix": "/data/process_dataset/test_merge/merge",41+ "output-prefix": "/data/ci/cache/process_dataset/test_merge/merge",
42 "merge-group-keys": "text_document"42 "merge-group-keys": "text_document"
43 }43 }
44 }44 }
Mtests/mindspore/st/shell_scripts/glm_pretrain.sh+3-3
@@ -16,9 +16,9 @@ NODE_RANK=0
16WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))16WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
17 17 
18 18 
19-DATA_PATH="/data/mindspore/st/test_glm_pretrain/dataset/dataset/alpaca_text_document"19+DATA_PATH="/data/ci/mindspore/st/test_glm_pretrain/dataset/dataset/alpaca_text_document"
20-TOKENIZER_PATH="/data/mindspore/st/test_glm_pretrain/tokenizer"20+TOKENIZER_PATH="/data/ci/mindspore/st/test_glm_pretrain/tokenizer"
21-CKPT_LOAD_DIR="/data/mindspore/st/test_glm_pretrain/ckpt_ut"21+CKPT_LOAD_DIR="/data/ci/mindspore/st/test_glm_pretrain/ckpt_ut"
22 22 
23TP=223TP=2
24PP=224PP=2
Mtests/pipeline/baichuan2-13B/baichuan2_13b_tp8_pp1_mcore.sh+3-3
@@ -13,9 +13,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
13basepath=$(cd `dirname $0`; cd ../../../; pwd)13basepath=$(cd `dirname $0`; cd ../../../; pwd)
14 14 
15 15 
16-DATA_PATH="/data/datasets/baichuan2-13B-data/enwiki/enwiki_text_document"16+DATA_PATH="/data/ci/datasets/processed/baichuan2-13B-data/enwiki/enwiki_text_document"
17-TOKENIZER_MODEL="/data/hf/baichuan2-13B-hf/tokenizer.model"17+TOKENIZER_MODEL="/data/ci/models/baichuan2/hf/baichuan2-13B-hf/tokenizer.model"
18-CKPT_LOAD_DIR="/data/pipeline/Baichuan2-13B-tp8pp1-mcore-hf-layer2/"18+CKPT_LOAD_DIR="/data/ci/models/baichuan2/mg/Baichuan2-13B-tp8pp1-mcore-hf-layer2/"
19 19 
20TP=820TP=8
21PP=121PP=1
Mtests/pipeline/baichuan2-13B/param_config.json+2-2
@@ -2,9 +2,9 @@
2 "test_preprocess_pretrain_data": [2 "test_preprocess_pretrain_data": [
3 {3 {
4 "param": {4 "param": {
5- "input": "/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet",5+ "input": "/data/ci/datasets/origin/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
6 "tokenizer-type": "PretrainedFromHF",6 "tokenizer-type": "PretrainedFromHF",
7- "tokenizer-name-or-path": "/data/hf/baichuan2-13B-hf/"7+ "tokenizer-name-or-path": "/data/ci/models/baichuan2/hf/baichuan2-13B-hf/"
8 },8 },
9 "prefix": "alpaca"9 "prefix": "alpaca"
10 }10 }
Mtests/pipeline/bloom-7B/param_config.json+2-2
@@ -2,9 +2,9 @@
2 "test_preprocess_pretrain_data": [2 "test_preprocess_pretrain_data": [
3 {3 {
4 "param": {4 "param": {
5- "input": "/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet",5+ "input": "/data/ci/datasets/origin/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
6 "tokenizer-type": "PretrainedFromHF",6 "tokenizer-type": "PretrainedFromHF",
7- "tokenizer-name-or-path": "/data/hf/bloom-7B-hf/"7+ "tokenizer-name-or-path": "/data/ci/models/bloom-7b/hf/bloom-7B-hf/"
8 },8 },
9 "prefix": "alpaca"9 "prefix": "alpaca"
10 }10 }
Mtests/pipeline/chatglm3-6B/chatglm3_tp1_pp2_rope.sh+3-3
@@ -11,9 +11,9 @@ WORLD_SIZE=$((NPUS_PER_NODE*$NNODES))
11basepath=$(cd `dirname $0`; cd ../../../; pwd)11basepath=$(cd `dirname $0`; cd ../../../; pwd)
12 12 
13CKPT_SAVE_DIR="/data/cache"13CKPT_SAVE_DIR="/data/cache"
14-DATA_PATH="/data/datasets/chatglm3-dataset-alpaca/alpaca_text_document"14+DATA_PATH="/data/ci/datasets/processed/chatglm3-dataset-alpaca/alpaca_text_document"
15-TOKENIZER_PATH="/data/hf/chatglm3-6b-base-hf/"15+TOKENIZER_PATH="/data/ci/models/chatglm3/hf/chatglm3-6b-base-hf/"
16-CKPT_LOAD_DIR="/data/pipeline/chatglm3-6b-base-mg-tp1pp2-mcore-base/"16+CKPT_LOAD_DIR="/data/ci/models/chatglm3/mg/chatglm3-6b-base-mg-tp1pp2-mcore-base/"
17 17 
18TP=118TP=1
19PP=219PP=2
Mtests/pipeline/chatglm3-6B/param_config.json+2-2
@@ -2,9 +2,9 @@
2 "test_preprocess_pretrain_data": [2 "test_preprocess_pretrain_data": [
3 {3 {
4 "param": {4 "param": {
5- "input": "/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet",5+ "input": "/data/ci/datasets/origin/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
6 "tokenizer-type": "PretrainedFromHF",6 "tokenizer-type": "PretrainedFromHF",
7- "tokenizer-name-or-path": "/data/hf/chatglm3-6b-base-hf/"7+ "tokenizer-name-or-path": "/data/ci/models/chatglm3/hf/chatglm3-6b-base-hf/"
8 },8 },
9 "prefix": "alpaca"9 "prefix": "alpaca"
10 }10 }
Mtests/pipeline/common/test_checkpoint.json+37-37
@@ -9,12 +9,12 @@
9 "target-pipeline-parallel-size": "2", 9 "target-pipeline-parallel-size": "2",
10 "target-expert-parallel-size": "2",10 "target-expert-parallel-size": "2",
11 "num-layer-list": "6,10",11 "num-layer-list": "6,10",
12- "load-dir":"/data/hf/Mixtral-legacy-hf",12+ "load-dir":"/data/ci/models/mixtral/hf/Mixtral-legacy-hf",
13- "save-dir":"/data/wttest/test/hf2mc_mixtral_tp2pp2ep2dypp",13+ "save-dir":"/data/ci/cache/hf2mc_mixtral_tp2pp2ep2dypp",
14 "use-mcore-models": null,14 "use-mcore-models": null,
15 "model-type-hf": "mixtral",15 "model-type-hf": "mixtral",
16 "params-dtype": "bf16",16 "params-dtype": "bf16",
17- "tokenizer-model":"/data/hf/Mixtral-legacy-hf/tokenizer.model"17+ "tokenizer-model":"/data/ci/models/mixtral/hf/Mixtral-legacy-hf/tokenizer.model"
18 }18 }
19 },19 },
20 {20 {
@@ -40,8 +40,8 @@
40 "target-tensor-parallel-size": "1",40 "target-tensor-parallel-size": "1",
41 "target-pipeline-parallel-size": "1",41 "target-pipeline-parallel-size": "1",
42 "target-expert-parallel-size": "1",42 "target-expert-parallel-size": "1",
43- "save-dir":"/data/hf/Mixtral-legacy-hf/",43+ "save-dir":"/data/ci/models/mixtral/hf/Mixtral-legacy-hf/",
44- "load-dir":"/data/wttest/base/hf2mc_mixtral_tp1pp4ep2vpp2",44+ "load-dir":"/data/ci/models/mixtral/mg/hf2mc_mixtral_tp1pp4ep2vpp2",
45 "use-mcore-models": null,45 "use-mcore-models": null,
46 "model-type-hf": "mixtral",46 "model-type-hf": "mixtral",
47 "params-dtype": "bf16"47 "params-dtype": "bf16"
@@ -72,12 +72,12 @@
72 "target-tensor-parallel-size": "1",72 "target-tensor-parallel-size": "1",
73 "target-pipeline-parallel-size": "1",73 "target-pipeline-parallel-size": "1",
74 "target-expert-parallel-size": "8",74 "target-expert-parallel-size": "8",
75- "load-dir":"/data/ci/deepseek2_lite/hf/deepseek2_lite_hf",75+ "load-dir":"/data/ci/models/deepseek2-lite/hf/deepseek2_lite_hf",
76- "save-dir":"/data/ci/deepseek2_lite/mg/deepseek2_lite_mg",76+ "save-dir":"/data/ci/cache/deepseek2_lite_mg",
77 "use-mcore-models": null,77 "use-mcore-models": null,
78 "model-type-hf": "deepseek2-lite",78 "model-type-hf": "deepseek2-lite",
79 "params-dtype": "bf16",79 "params-dtype": "bf16",
80- "tokenizer-model":"/data/ci/deepseek2_lite/hf/deepseek2_lite_hf",80+ "tokenizer-model":"/data/ci/models/deepseek2-lite/hf/deepseek2_lite_hf",
81 "spec":"mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec"81 "spec":"mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec"
82 }82 }
83 },83 },
@@ -104,8 +104,8 @@
104 "target-tensor-parallel-size": "1",104 "target-tensor-parallel-size": "1",
105 "target-pipeline-parallel-size": "1",105 "target-pipeline-parallel-size": "1",
106 "target-expert-parallel-size": "1",106 "target-expert-parallel-size": "1",
107- "save-dir":"/data/ci/deepseek2_lite/hf/deepseek2_lite_hf",107+ "save-dir":"/data/ci/models/deepseek2-lite/hf/deepseek2_lite_hf",
108- "load-dir":"/data/pipeline/deepseek2_lite_mg_v2/",108+ "load-dir":"/data/ci/models/deepseek2-lite/mg/deepseek2_lite_mg_v2/",
109 "use-mcore-models": null,109 "use-mcore-models": null,
110 "model-type-hf": "deepseek2-lite",110 "model-type-hf": "deepseek2-lite",
111 "params-dtype": "bf16",111 "params-dtype": "bf16",
@@ -127,13 +127,13 @@
127 "save-model-type": "mg",127 "save-model-type": "mg",
128 "target-tensor-parallel-size": "8",128 "target-tensor-parallel-size": "8",
129 "target-pipeline-parallel-size": "1",129 "target-pipeline-parallel-size": "1",
130- "load-dir": "/data/hf/gemma2-9b-hf/",130+ "load-dir": "/data/ci/models/gemma2/hf/gemma2-9b-hf/",
131- "save-dir": "/data/gemma2-9b-mg-tp8pp1-mcore-test/",131+ "save-dir": "/data/ci/cache/gemma2-9b-mg-tp8pp1-mcore-test/",
132 "use-mcore-models": null,132 "use-mcore-models": null,
133 "post-norm": null,133 "post-norm": null,
134 "model-type-hf": "gemma2",134 "model-type-hf": "gemma2",
135 "params-dtype": "bf16",135 "params-dtype": "bf16",
136- "tokenizer-model": "/data/hf/gemma2-9b-hf/tokenizer.json",136+ "tokenizer-model": "/data/ci/models/gemma2/hf/gemma2-9b-hf/tokenizer.json",
137 "spec":"mindspeed_llm.tasks.models.spec.gemma2_spec layer_spec"137 "spec":"mindspeed_llm.tasks.models.spec.gemma2_spec layer_spec"
138 }138 }
139 },139 },
@@ -157,9 +157,9 @@
157 "model-type":"GPT",157 "model-type":"GPT",
158 "load-model-type":"mg",158 "load-model-type":"mg",
159 "save-model-type":"mg",159 "save-model-type":"mg",
160- "load-dir":"/data/llama2_ckpt_lora_UT/Llama2-mcore/",160+ "load-dir":"/data/ci/models/llama2/mg/Llama2-mcore/",
161- "save-dir":"/data/llama2_ckpt_lora_UT/llama2-7b-lora2mcore-test/",161+ "save-dir":"/data/ci/cache/llama2-7b-lora2mcore-test/",
162- "lora-load":"/data/llama2_ckpt_lora_UT/llama-7b-lora-mcore-tp1pp1/",162+ "lora-load":"/data/ci/models/llama2/mg/llama-7b-lora-mcore-tp1pp1/",
163 "lora-r":"16",163 "lora-r":"16",
164 "lora-alpha":"32",164 "lora-alpha":"32",
165 "lora-target-modules":"linear_qkv linear_proj linear_fc1 linear_fc2",165 "lora-target-modules":"linear_qkv linear_proj linear_fc1 linear_fc2",
@@ -185,9 +185,9 @@
185 "target-tensor-parallel-size": "1",185 "target-tensor-parallel-size": "1",
186 "target-pipeline-parallel-size": "2",186 "target-pipeline-parallel-size": "2",
187 "target-expert-parallel-size": "2",187 "target-expert-parallel-size": "2",
188- "load-dir":"/data/ci/qwen2_moe/hf_base/qwen2_moe_hf",188+ "load-dir":"/data/ci/models/qwen2-moe/hf/qwen2_moe_hf",
189- "save-dir":"/data/ci/qwen2_moe/mg_base/qwen2_moe_l2_t1p2e2",189+ "save-dir":"/data/ci/cache/qwen2_moe_l2_t1p2e2",
190- "tokenizer-model":"/data/ci/qwen2_moe/hf_base/qwen2_moe_hf",190+ "tokenizer-model":"/data/ci/models/qwen2-moe/hf/qwen2_moe_hf",
191 "use-mcore-models": null,191 "use-mcore-models": null,
192 "moe-grouped-gemm": null,192 "moe-grouped-gemm": null,
193 "add-qkv-bias": null,193 "add-qkv-bias": null,
@@ -215,8 +215,8 @@
215 "target-tensor-parallel-size": "1",215 "target-tensor-parallel-size": "1",
216 "target-pipeline-parallel-size": "1",216 "target-pipeline-parallel-size": "1",
217 "target-expert-parallel-size": "1",217 "target-expert-parallel-size": "1",
218- "save-dir":"/data/ci/qwen2_moe/hf_base/qwen2_moe_hf",218+ "save-dir":"/data/ci/models/qwen2-moe/hf/qwen2_moe_hf",
219- "load-dir":"/data/ci/qwen2_moe/mg_base/qwen2_moe_l2_t1p2e2",219+ "load-dir":"/data/ci/cache/qwen2_moe_l2_t1p2e2",
220 "spec":"mindspeed_llm.tasks.models.spec.qwen2_moe_spec layer_spec",220 "spec":"mindspeed_llm.tasks.models.spec.qwen2_moe_spec layer_spec",
221 "use-mcore-models": null,221 "use-mcore-models": null,
222 "moe-grouped-gemm": null,222 "moe-grouped-gemm": null,
@@ -241,8 +241,8 @@
241 "target-tensor-parallel-size": "1",241 "target-tensor-parallel-size": "1",
242 "target-pipeline-parallel-size": "1",242 "target-pipeline-parallel-size": "1",
243 "target-expert-parallel-size": "1",243 "target-expert-parallel-size": "1",
244- "save-dir":"/data/ci/qwen2_moe/hf_base/qwen2_moe_hf",244+ "save-dir":"/data/ci/models/qwen2-moe/hf/qwen2_moe_hf",
245- "load-dir":"/data/ci/qwen2_moe/mg_base/qwen2_moe_l2_t2p1e2_base",245+ "load-dir":"/data/ci/models/qwen2-moe/mg/qwen2_moe_l2_t2p1e2_base",
246 "spec":"mindspeed_llm.tasks.models.spec.qwen2_moe_spec layer_spec",246 "spec":"mindspeed_llm.tasks.models.spec.qwen2_moe_spec layer_spec",
247 "use-mcore-models": null,247 "use-mcore-models": null,
248 "moe-grouped-gemm": null,248 "moe-grouped-gemm": null,
@@ -268,12 +268,12 @@
268 "save-model-type": "mg",268 "save-model-type": "mg",
269 "target-tensor-parallel-size": "2",269 "target-tensor-parallel-size": "2",
270 "target-pipeline-parallel-size": "2",270 "target-pipeline-parallel-size": "2",
271- "save-dir":"/data/llama-3-8b-hf-nooplayer-tp2pp2vpp2-mcore-test/",271+ "save-dir":"/data/ci/cache/llama-3-8b-hf-nooplayer-tp2pp2vpp2-mcore-test/",
272- "load-dir":"/data/hf/llama-3-8b-hf-layer14/",272+ "load-dir":"/data/ci/models/llama3/hf/llama-3-8b-hf-layer14/",
273 "num-layers-per-virtual-pipeline-stage": "2",273 "num-layers-per-virtual-pipeline-stage": "2",
274 "noop-layers": "1,15",274 "noop-layers": "1,15",
275 "params-dtype": "bf16",275 "params-dtype": "bf16",
276- "tokenizer-model": "/data/llama-3-8b-hf/tokenizer.json"276+ "tokenizer-model": "/data/ci/models/llama3/hf/llama-3-8b-hf/tokenizer.json"
277 }277 }
278 },278 },
279 {279 {
@@ -295,10 +295,10 @@
295 "save-model-type":"mg",295 "save-model-type":"mg",
296 "target-tensor-parallel-size": "1",296 "target-tensor-parallel-size": "1",
297 "target-pipeline-parallel-size": "1",297 "target-pipeline-parallel-size": "1",
298- "load-dir":"/data/Qwen2-1.5B/",298+ "load-dir":"/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B/",
299- "save-dir":"/data/qwen2-1.5b-base-mg-tp1pp1-mcore-test/",299+ "save-dir":"/data/ci/cache/qwen2-1.5b-base-mg-tp1pp1-mcore-test/",
300 "model-type-hf": "llama2",300 "model-type-hf": "llama2",
301- "tokenizer-model":"/data/Qwen2-1.5B/tokenizer.model",301+ "tokenizer-model":"/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B/tokenizer.model",
302 "add-qkv-bias": null,302 "add-qkv-bias": null,
303 "params-dtype": "bf16"303 "params-dtype": "bf16"
304 }304 }
@@ -378,8 +378,8 @@
378 "params-dtype":"bf16",378 "params-dtype":"bf16",
379 "target-tensor-parallel-size": "1",379 "target-tensor-parallel-size": "1",
380 "target-pipeline-parallel-size": "1",380 "target-pipeline-parallel-size": "1",
381- "load-dir":"/data/ci/orm/llama-2-7b-layers8-rm-mcore_pp2vpp2/",381+ "load-dir":"/data/ci/models/llama2/mg/llama-2-7b-layers8-rm-mcore_pp2vpp2/",
382- "save-dir":"/data/ci/orm/llama-2-7b-layers8-rm-hf/",382+ "save-dir":"/data/ci/models/llama2/hf/llama-2-7b-layers8-rm-hf/",
383 "use-mcore-models": null,383 "use-mcore-models": null,
384 "model-type-hf": "llama2",384 "model-type-hf": "llama2",
385 "orm": null385 "orm": null
@@ -401,9 +401,9 @@
401 "target-tensor-parallel-size": "2",401 "target-tensor-parallel-size": "2",
402 "target-pipeline-parallel-size": "1",402 "target-pipeline-parallel-size": "1",
403 "target-expert-parallel-size": "2",403 "target-expert-parallel-size": "2",
404- "load-dir":"/data/ci/qwen2_moe/hf_base/qwen2_moe_hf",404+ "load-dir":"/data/ci/models/qwen2-moe/hf/qwen2_moe_hf",
405- "save-dir":"/data/qwen2_moe_tp2_ep2_extend_mc",405+ "save-dir":"/data/ci/cache/qwen2_moe_tp2_ep2_extend_mc",
406- "tokenizer-model":"/data/ci/qwen2_moe/hf_base/qwen2_moe_hf",406+ "tokenizer-model":"/data/ci/models/qwen2-moe/hf/qwen2_moe_hf",
407 "spec":"mindspeed_llm.tasks.models.spec.qwen2_moe_spec layer_spec",407 "spec":"mindspeed_llm.tasks.models.spec.qwen2_moe_spec layer_spec",
408 "use-mcore-models": null,408 "use-mcore-models": null,
409 "moe-grouped-gemm": null,409 "moe-grouped-gemm": null,
@@ -432,9 +432,9 @@
432 "target-tensor-parallel-size": "2",432 "target-tensor-parallel-size": "2",
433 "target-pipeline-parallel-size": "1",433 "target-pipeline-parallel-size": "1",
434 "target-expert-parallel-size": "2",434 "target-expert-parallel-size": "2",
435- "load-dir":"/data/ci/qwen2_moe/hf_base/qwen2_moe_hf",435+ "load-dir":"/data/ci/models/qwen2-moe/hf/qwen2_moe_hf",
436- "save-dir":"/data/ci/qwen2_moe/mg_base/qwen2_moe_l2_t2p1e2",436+ "save-dir":"/data/ci/cache/qwen2_moe_l2_t2p1e2",
437- "tokenizer-model":"/data/ci/qwen2_moe/hf_base/qwen2_moe_hf",437+ "tokenizer-model":"/data/ci/models/qwen2-moe/hf/qwen2_moe_hf",
438 "spec":"mindspeed_llm.tasks.models.spec.qwen2_moe_spec layer_spec",438 "spec":"mindspeed_llm.tasks.models.spec.qwen2_moe_spec layer_spec",
439 "use-mcore-models": null,439 "use-mcore-models": null,
440 "moe-grouped-gemm": null,440 "moe-grouped-gemm": null,
Mtests/pipeline/common/test_checkpoint.py+1-1
@@ -194,7 +194,7 @@ class TestCheckpoint(object):
194 os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "1"194 os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "1"
195 exit_code = run_cmd(["python3", CKPT_PYPATH] + self.test_config_cmd['test_qwen2_moe_hf2mcore_tp2pp1ep2'])195 exit_code = run_cmd(["python3", CKPT_PYPATH] + self.test_config_cmd['test_qwen2_moe_hf2mcore_tp2pp1ep2'])
196 assert exit_code == 0196 assert exit_code == 0
197- base_dir = "/data/ci/qwen2_moe/mg_base/qwen2_moe_l2_t2p1e2_base"197+ base_dir = "/data/ci/models/qwen2-moe/mg/qwen2_moe_l2_t2p1e2_base"
198 save_dir = self.test_config['test_qwen2_moe_hf2mcore_tp2pp1ep2'][0]['save-dir']198 save_dir = self.test_config['test_qwen2_moe_hf2mcore_tp2pp1ep2'][0]['save-dir']
199 assert weight_compare(base_dir, save_dir)199 assert weight_compare(base_dir, save_dir)
200 shutil.rmtree(save_dir)200 shutil.rmtree(save_dir)
Mtests/pipeline/common/test_evaluate.json+3-3
@@ -2,7 +2,7 @@
2 "test_llama2_mcore_humaneval_evaluate": [2 "test_llama2_mcore_humaneval_evaluate": [
3 {3 {
4 "param": {4 "param": {
5- "task-data-path":"/data/eval_data/human_eval",5+ "task-data-path":"/data/ci/datasets/eval_data/human_eval",
6 "task":"human_eval",6 "task":"human_eval",
7 "tensor-model-parallel-size": 8,7 "tensor-model-parallel-size": 8,
8 "pipeline-model-parallel-size": 1,8 "pipeline-model-parallel-size": 1,
@@ -27,9 +27,9 @@
27 "fp16": null,27 "fp16": null,
28 "no-load-rng": null,28 "no-load-rng": null,
29 "no-load-optim": null,29 "no-load-optim": null,
30- "load":"/data/ci/ckpt",30+ "load":"/data/ci/models/llama2/mg/llama2-tp8pp1",
31 "tokenizer-type":"PretrainedFromHF",31 "tokenizer-type":"PretrainedFromHF",
32- "tokenizer-name-or-path":"/data/hf/llama-2-7b-hf"32+ "tokenizer-name-or-path":"/data/ci/models/llama2/hf/llama-2-7b-hf"
33 }33 }
34 }34 }
35 ]35 ]
Mtests/pipeline/common/test_inference.json+5-5
@@ -29,10 +29,10 @@
29 "max-new-tokens": 30,29 "max-new-tokens": 30,
30 "micro-batch-size": 1,30 "micro-batch-size": 1,
31 "global-batch-size": 1,31 "global-batch-size": 1,
32- "load":"/data/pipeline/chatglm3-6b-base-mg-tp1pp2-mcore-base",32+ "load":"/data/ci/models/chatglm3/mg/chatglm3-6b-base-mg-tp1pp2-mcore-base",
33 "tokenizer-type": "PretrainedFromHF",33 "tokenizer-type": "PretrainedFromHF",
34- "tokenizer-name-or-path":"/data/hf/chatglm3-6b-base-hf/",34+ "tokenizer-name-or-path":"/data/ci/models/chatglm3/hf/chatglm3-6b-base-hf/",
35- "tokenizer-model": "/data/hf/chatglm3-6b-base-hf/tokenizer.model",35+ "tokenizer-model": "/data/ci/models/chatglm3/hf/chatglm3-6b-base-hf/tokenizer.model",
36 "tokenizer-not-use-fast": null,36 "tokenizer-not-use-fast": null,
37 "untie-embeddings-and-output-weights": null,37 "untie-embeddings-and-output-weights": null,
38 "attention-softmax-in-fp32": null,38 "attention-softmax-in-fp32": null,
@@ -69,9 +69,9 @@
69 "square-alibi-mask": null,69 "square-alibi-mask": null,
70 "fill-neg-inf": null,70 "fill-neg-inf": null,
71 "swiglu": null,71 "swiglu": null,
72- "load": "/data/Baichuan2-13B-tp8pp1-mcore-hf",72+ "load": "/data/ci/models/baichuan2/mg/Baichuan2-13B-tp8pp1-mcore-hf",
73 "tokenizer-type": "PretrainedFromHF",73 "tokenizer-type": "PretrainedFromHF",
74- "tokenizer-name-or-path": "/data/hf/baichuan2-13B-hf/",74+ "tokenizer-name-or-path": "/data/ci/models/baichuan2/hf/baichuan2-13B-hf/",
75 "tokenizer-not-use-fast": null,75 "tokenizer-not-use-fast": null,
76 "fp16": null,76 "fp16": null,
77 "normalization": "RMSNorm" ,77 "normalization": "RMSNorm" ,
Mtests/pipeline/common/test_process_instruction_data_lf.json+17-17
@@ -2,12 +2,12 @@
2 "test_alpaca_dataset": [2 "test_alpaca_dataset": [
3 {3 {
4 "params": {4 "params": {
5- "input": "/data/tune_dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet",5+ "input": "/data/ci/datasets/tune_dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
6 "tokenizer-type": "PretrainedFromHF",6 "tokenizer-type": "PretrainedFromHF",
7 "handler-name": "AlpacaStyleInstructionHandler",7 "handler-name": "AlpacaStyleInstructionHandler",
8 "output-prefix": "/data/cache/tune_dataset/alpaca/alpaca",8 "output-prefix": "/data/cache/tune_dataset/alpaca/alpaca",
9 "overwrite-cache": null,9 "overwrite-cache": null,
10- "tokenizer-name-or-path": "/data/qwen-7b/",10+ "tokenizer-name-or-path": "/data/ci/models/qwen-7b/hf/qwen-7b/",
11 "workers": 4,11 "workers": 4,
12 "log-interval": 1000,12 "log-interval": 1000,
13 "prompt-type": "qwen"13 "prompt-type": "qwen"
@@ -17,11 +17,11 @@
17 "test_alpaca_history_dataset": [17 "test_alpaca_history_dataset": [
18 {18 {
19 "params": {19 "params": {
20- "input": "/data/tune_dataset/oaast_sft.json",20+ "input": "/data/ci/datasets/tune_dataset/oaast_sft.json",
21 "tokenizer-type": "PretrainedFromHF",21 "tokenizer-type": "PretrainedFromHF",
22 "handler-name": "AlpacaStyleInstructionHandler",22 "handler-name": "AlpacaStyleInstructionHandler",
23- "output-prefix": "/data/tune_dataset/alpaca_his/alpaca_his",23+ "output-prefix": "/data/ci/datasets/tune_dataset/alpaca_his/alpaca_his",
24- "tokenizer-name-or-path": "/data/qwen-7b/",24+ "tokenizer-name-or-path": "/data/ci/models/qwen-7b/hf/qwen-7b/",
25 "overwrite-cache": null,25 "overwrite-cache": null,
26 "workers": 4,26 "workers": 4,
27 "log-interval": 1000,27 "log-interval": 1000,
@@ -31,11 +31,11 @@
31 },31 },
32 {32 {
33 "params": {33 "params": {
34- "input": "/data/tune_dataset/oaast_sft.json",34+ "input": "/data/ci/datasets/tune_dataset/oaast_sft.json",
35 "tokenizer-type": "PretrainedFromHF",35 "tokenizer-type": "PretrainedFromHF",
36 "handler-name": "AlpacaStyleInstructionHandler",36 "handler-name": "AlpacaStyleInstructionHandler",
37- "output-prefix": "/data/tune_dataset/alpaca_his/alpaca_his_seq1024",37+ "output-prefix": "/data/ci/datasets/tune_dataset/alpaca_his/alpaca_his_seq1024",
38- "tokenizer-name-or-path": "/data/qwen-7b/",38+ "tokenizer-name-or-path": "/data/ci/models/qwen-7b/hf/qwen-7b/",
39 "overwrite-cache": null,39 "overwrite-cache": null,
40 "workers": 4,40 "workers": 4,
41 "log-interval": 1000,41 "log-interval": 1000,
@@ -49,11 +49,11 @@
49 "test_sharegpt_dataset": [49 "test_sharegpt_dataset": [
50 {50 {
51 "params": {51 "params": {
52- "input": "/data/tune_dataset/sharegpt_formatted_data-evol-gpt4.jsonl",52+ "input": "/data/ci/datasets/tune_dataset/sharegpt_formatted_data-evol-gpt4.jsonl",
53 "tokenizer-type": "PretrainedFromHF",53 "tokenizer-type": "PretrainedFromHF",
54 "handler-name": "SharegptStyleInstructionHandler",54 "handler-name": "SharegptStyleInstructionHandler",
55- "output-prefix": "/data/tune_dataset/sharegpt/sharegpt",55+ "output-prefix": "/data/ci/datasets/tune_dataset/sharegpt/sharegpt",
56- "tokenizer-name-or-path": "/data/qwen-7b/",56+ "tokenizer-name-or-path": "/data/ci/models/qwen-7b/hf/qwen-7b/",
57 "workers": 4,57 "workers": 4,
58 "overwrite-cache": null,58 "overwrite-cache": null,
59 "log-interval": 1000,59 "log-interval": 1000,
@@ -65,11 +65,11 @@
65 "test_openai_dataset": [65 "test_openai_dataset": [
66 {66 {
67 "params": {67 "params": {
68- "input": "/data/tune_dataset/sss.json",68+ "input": "/data/ci/datasets/tune_dataset/sss.json",
69 "tokenizer-type": "PretrainedFromHF",69 "tokenizer-type": "PretrainedFromHF",
70 "handler-name": "SharegptStyleInstructionHandler",70 "handler-name": "SharegptStyleInstructionHandler",
71- "output-prefix": "/data/tune_dataset/sharegpt/sharegpt",71+ "output-prefix": "/data/ci/datasets/tune_dataset/sharegpt/sharegpt",
72- "tokenizer-name-or-path": "/data/qwen-7b/",72+ "tokenizer-name-or-path": "/data/ci/models/qwen-7b/hf/qwen-7b/",
73 "overwrite-cache": null,73 "overwrite-cache": null,
74 "workers": 4,74 "workers": 4,
75 "log-interval": 1000,75 "log-interval": 1000,
@@ -81,11 +81,11 @@
81 "test_abstract_prompt_type": [81 "test_abstract_prompt_type": [
82 {82 {
83 "params": {83 "params": {
84- "input": "/data/tune_dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet",84+ "input": "/data/ci/datasets/tune_dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
85 "tokenizer-type": "PretrainedFromHF",85 "tokenizer-type": "PretrainedFromHF",
86 "handler-name": "AlpacaStyleInstructionHandler",86 "handler-name": "AlpacaStyleInstructionHandler",
87- "output-prefix": "/data/tune_dataset/prompt_abstract/alpaca",87+ "output-prefix": "/data/ci/datasets/tune_dataset/prompt_abstract/alpaca",
88- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf/",88+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf/",
89 "overwrite-cache": null,89 "overwrite-cache": null,
90 "workers": 4,90 "workers": 4,
91 "log-interval": 1000,91 "log-interval": 1000,
Mtests/pipeline/common/test_process_instruction_data_lf.py+5-5
@@ -17,10 +17,10 @@ class TestProcessInstructionDataLf:
17 17 
18 @pytest.mark.parametrize("params, base_path", 18 @pytest.mark.parametrize("params, base_path",
19 [19 [
20- (test_config["test_alpaca_dataset"][0], "/data/tune_dataset/Llamafactoryhandler/alpaca/alpaca"),20+ (test_config["test_alpaca_dataset"][0], "/data/ci/datasets/tune_dataset/Llamafactoryhandler/alpaca/alpaca"),
21- (test_config["test_alpaca_history_dataset"][0], "/data/tune_dataset/Llamafactoryhandler/alpaca_history/alpaca_history_new"),21+ (test_config["test_alpaca_history_dataset"][0], "/data/ci/datasets/tune_dataset/Llamafactoryhandler/alpaca_history/alpaca_history_new"),
22- (test_config["test_sharegpt_dataset"][0], "/data/tune_dataset/Llamafactoryhandler/sharegpt/sharegpt_lf"),22+ (test_config["test_sharegpt_dataset"][0], "/data/ci/datasets/tune_dataset/Llamafactoryhandler/sharegpt/sharegpt_lf"),
23- (test_config["test_openai_dataset"][0], "/data/tune_dataset/Llamafactoryhandler/openai/sss"),23+ (test_config["test_openai_dataset"][0], "/data/ci/datasets/tune_dataset/Llamafactoryhandler/openai/sss"),
24 (test_config["test_abstract_prompt_type"][0], "/data/Llama2-7b-original-prompt-type/alpaca")24 (test_config["test_abstract_prompt_type"][0], "/data/Llama2-7b-original-prompt-type/alpaca")
25 ])25 ])
26 def test_datasets(self, build_args, params, base_path):26 def test_datasets(self, build_args, params, base_path):
@@ -62,7 +62,7 @@ class TestProcessInstructionDataLf:
62 62 
63 @pytest.mark.parametrize("params, base_path", 63 @pytest.mark.parametrize("params, base_path",
64 [64 [
65- (test_config["test_alpaca_history_dataset"][1], "/data/tune_dataset/Llamafactoryhandler/alpaca_history/alpaca_history_seq1024"),65+ (test_config["test_alpaca_history_dataset"][1], "/data/ci/datasets/tune_dataset/Llamafactoryhandler/alpaca_history/alpaca_history_seq1024"),
66 ])66 ])
67 def test_skip_num(self, build_args, params, base_path):67 def test_skip_num(self, build_args, params, base_path):
68 """68 """
Mtests/pipeline/common/test_process_instruction_pack_data.py+3-3
@@ -14,11 +14,11 @@ class TestProcessInstructionData:
14 def setup_class(self):14 def setup_class(self):
15 sys.argv = [15 sys.argv = [
16 sys.argv[0],16 sys.argv[0],
17- "--input", "/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet",17+ "--input", "/data/ci/datasets/origin/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
18 "--tokenizer-type", "PretrainedFromHF",18 "--tokenizer-type", "PretrainedFromHF",
19 "--handler-name", "GeneralInstructionHandler",19 "--handler-name", "GeneralInstructionHandler",
20- "--output-prefix", "/data/tune_pack_dataset/alpaca_pack",20+ "--output-prefix", "/data/ci/cache/tune_pack_dataset/alpaca_pack",
21- "--tokenizer-name-or-path", "/data/hf/llama-2-7b-hf",21+ "--tokenizer-name-or-path", "/data/ci/models/llama2/hf/llama-2-7b-hf",
22 "--workers", "4",22 "--workers", "4",
23 "--log-interval", "1000",23 "--log-interval", "1000",
24 "--append-eod",24 "--append-eod",
Mtests/pipeline/common/test_process_pairwise_data_lf.json+8-8
@@ -2,32 +2,32 @@
2 "test_alpaca_dataset": [2 "test_alpaca_dataset": [
3 {3 {
4 "params": {4 "params": {
5- "input": "/data/pairwise_dataset/orca_rlhf.jsonl",5+ "input": "/data/ci/datasets/origin/pairwise_dataset/orca_rlhf.jsonl",
6 "tokenizer-type": "PretrainedFromHF",6 "tokenizer-type": "PretrainedFromHF",
7- "output-prefix": "/data/pairwise_dataset/output/orca_rlhf/orca_rlhf", 7+ "output-prefix": "/data/ci/datasets/origin/pairwise_dataset/output/orca_rlhf/orca_rlhf",
8- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf/",8+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf/",
9 "workers": 4,9 "workers": 4,
10 "log-interval": 1000,10 "log-interval": 1000,
11 "handler-name": "AlpacaStylePairwiseHandler",11 "handler-name": "AlpacaStylePairwiseHandler",
12 "prompt-type": "llama2",12 "prompt-type": "llama2",
13 "map-keys": "{\"prompt\":\"question\", \"query\":\"\", \"system\":\"system\"}"13 "map-keys": "{\"prompt\":\"question\", \"query\":\"\", \"system\":\"system\"}"
14 },14 },
15- "baseline": "/data/pairwise_dataset/baseline/orca_rlhf/orca_rlhf"15+ "baseline": "/data/ci/datasets/origin/pairwise_dataset/baseline/orca_rlhf/orca_rlhf"
16 }16 }
17 ],17 ],
18 "test_sharegpt_dataset": [18 "test_sharegpt_dataset": [
19 {19 {
20 "params": {20 "params": {
21- "input": "/data/pairwise_dataset/dpo_en.json",21+ "input": "/data/ci/datasets/origin/pairwise_dataset/dpo_en.json",
22 "tokenizer-type": "PretrainedFromHF",22 "tokenizer-type": "PretrainedFromHF",
23 "handler-name": "SharegptStylePairwiseHandler",23 "handler-name": "SharegptStylePairwiseHandler",
24- "output-prefix": "/data/pairwise_dataset/output/dpo_en/dpo_en",24+ "output-prefix": "/data/ci/datasets/origin/pairwise_dataset/output/dpo_en/dpo_en",
25- "tokenizer-name-or-path": "/data/hf/chatglm3-6b-base-hf/",25+ "tokenizer-name-or-path": "/data/ci/models/chatglm3/hf/chatglm3-6b-base-hf/",
26 "workers": 4,26 "workers": 4,
27 "log-interval": 1000,27 "log-interval": 1000,
28 "prompt-type": "chatglm3"28 "prompt-type": "chatglm3"
29 },29 },
30- "baseline": "/data/pairwise_dataset/baseline/dpo_en/dpo_en"30+ "baseline": "/data/ci/datasets/origin/pairwise_dataset/baseline/dpo_en/dpo_en"
31 }31 }
32 ]32 ]
33}33}
Mtests/pipeline/deepseek/deepseek2_tp1_pp1_mcore_moe.sh+2-2
@@ -12,8 +12,8 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
12basepath=$(cd `dirname $0`; cd ../../../; pwd)12basepath=$(cd `dirname $0`; cd ../../../; pwd)
13 13 
14DATA_PATH="/data/datasets/deepseek2_tp1_pp1_moe_enwiki/enwiki_text_document"14DATA_PATH="/data/datasets/deepseek2_tp1_pp1_moe_enwiki/enwiki_text_document"
15-TOKENIZER_MODEL=" /data/hf/deepseek2_tp1_pp1_moe_hf "15+TOKENIZER_MODEL=" /data/ci/models/deepseek-v2/hf/deepseek2_tp1_pp1_moe_hf "
16-CKPT_LOAD_DIR="/data/pipeline/deepseek2_tp1_pp1_ckpt/"16+CKPT_LOAD_DIR="/data/ci/models/deepseek-v2/mg/deepseek2_tp1_pp1_ckpt/"
17CKPT_SAVE_DIR="/data/cache"17CKPT_SAVE_DIR="/data/cache"
18 18 
19TP=119TP=1
Mtests/pipeline/gemma-7B/gemma2_tp8_pp1_ptd.sh+2-2
@@ -13,8 +13,8 @@ basepath=$(cd `dirname $0`; cd ../../../; pwd)
13 13 
14CKPT_SAVE_DIR="/data/cache"14CKPT_SAVE_DIR="/data/cache"
15DATA_PATH="/data/datasets/gemma2-dataset-alpaca/alpaca_text_document"15DATA_PATH="/data/datasets/gemma2-dataset-alpaca/alpaca_text_document"
16-TOKENIZER_MODEL="/data/hf/gemma2-9b-hf"16+TOKENIZER_MODEL="/data/ci/models/gemma2/hf/gemma2-9b-hf"
17-CKPT_LOAD_DIR="/data/pipeline/gemma2-9b-mg-tp8pp1-mcore-base"17+CKPT_LOAD_DIR="/data/ci/models/gemma2/mg/gemma2-9b-mg-tp8pp1-mcore-base"
18 18 
19TP=819TP=8
20PP=120PP=1
Mtests/pipeline/gemma-7B/param_config.json+2-2
@@ -2,9 +2,9 @@
2 "test_preprocess_pretrain_data": [2 "test_preprocess_pretrain_data": [
3 {3 {
4 "param": {4 "param": {
5- "input": "/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet",5+ "input": "/data/ci/datasets/origin/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
6 "tokenizer-type": "PretrainedFromHF",6 "tokenizer-type": "PretrainedFromHF",
7- "tokenizer-name-or-path": "/data/hf/gemma-7b-hf/"7+ "tokenizer-name-or-path": "/data/ci/models/gemma/hf/gemma-7b-hf/"
8 },8 },
9 "prefix": "alpaca"9 "prefix": "alpaca"
10 }10 }
Mtests/pipeline/glm4/glm4_9b_8k_tp2_pp2_ptd.sh+2-2
@@ -10,8 +10,8 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
10 10 
11CKPT_SAVE_DIR="/data/cache"11CKPT_SAVE_DIR="/data/cache"
12DATA_PATH="/data/datasets/glm4_9b_dataset/alpaca_text_document"12DATA_PATH="/data/datasets/glm4_9b_dataset/alpaca_text_document"
13-TOKENIZER_PATH="/data/hf/glm4_9b_hf"13+TOKENIZER_PATH="/data/ci/models/glm4/hf/glm4_9b_hf"
14-CKPT_LOAD_DIR="/data/pipeline/glm4_tp2pp2"14+CKPT_LOAD_DIR="/data/ci/models/glm4/mg/glm4_tp2pp2"
15 15 
16TP=216TP=2
17PP=217PP=2
Mtests/pipeline/gpt4/gpt4_mcore_tp4_cp2_32k_moe_drop.sh+4-4
@@ -11,10 +11,10 @@ WORLD_SIZE=$((NPUS_PER_NODE*$NNODES))
11 11 
12basepath=$(cd `dirname $0`; cd ../../../; pwd)12basepath=$(cd `dirname $0`; cd ../../../; pwd)
13 13 
14-DATA_PATH=/data/datasets/gpt/gpt-wiki_text_document14+DATA_PATH=/data/ci/datasets/processed/gpt/gpt-wiki_text_document
15-VOCAB_FILE=/data/datasets/gpt/gpt2-vocab.json15+VOCAB_FILE=/data/ci/datasets/processed/gpt/gpt2-vocab.json
16-MERGE_FILE=/data/datasets/gpt/gpt2-merges.txt16+MERGE_FILE=/data/ci/datasets/processed/gpt/gpt2-merges.txt
17-CKPT_LOAD_DIR=/data/pipeline/gpt_drop_moe_32k_base17+CKPT_LOAD_DIR=/data/ci/models/gpt/mg/gpt_drop_moe_32k_base
18CKPT_SAVE_DIR=/data/cache18CKPT_SAVE_DIR=/data/cache
19 19 
20TP=420TP=4
Mtests/pipeline/grok1/grok1_40b_tp4_ep2_ptd.sh+3-3
@@ -14,9 +14,9 @@ NODE_RANK=0
14WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))14WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
15 15 
16CKPT_SAVE_DIR="/data/cache"16CKPT_SAVE_DIR="/data/cache"
17-CKPT_LOAD_DIR="/data/pipeline/grok1_tp1pp1ep2"17+CKPT_LOAD_DIR="/data/ci/models/grok1/mg/grok1_tp1pp1ep2"
18-DATA_PATH="/data/datasets/grok1/enwiki_text_document"18+DATA_PATH="/data/ci/datasets/processed/grok1/enwiki_text_document"
19-TOKENIZER_PATH="/data/hf/grok1_hf"19+TOKENIZER_PATH="/data/ci/models/grok1/hf/grok1_hf"
20 20 
21TP=421TP=4
22PP=122PP=1
Mtests/pipeline/hunyuan/tune_hunyuanLarge_389b_tp1_pp1_ep8_ptd.sh+3-3
@@ -12,10 +12,10 @@ NNODES=1
12NODE_RANK=012NODE_RANK=0
13WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))13WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
14 14 
15-TOKENIZER_PATH="/data/hf/Hunyuan-A52B-Instruct"15+TOKENIZER_PATH="/data/ci/models/hunyuan/hf/Hunyuan-A52B-Instruct"
16-CKPT_LOAD_DIR="/data/pipeline/hunyuanlarge_tp1pp1ep8"16+CKPT_LOAD_DIR="/data/ci/models/hunyuan/mg/hunyuanlarge_tp1pp1ep8"
17CKPT_SAVE_DIR="/data/cache"17CKPT_SAVE_DIR="/data/cache"
18-DATA_PATH="/data/datasets/hunyuan_finetune/hunyuan"18+DATA_PATH="/data/ci/datasets/processed/hunyuan_finetune/hunyuan"
19 19 
20TP=120TP=1
21PP=121PP=1
Mtests/pipeline/interlm3/internlm3_8b_tp1_pp4_cp2_ptd.sh+1-1
@@ -11,7 +11,7 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
11 11 
12# please fill these path configurations12# please fill these path configurations
13CKPT_SAVE_DIR="/data/cache"13CKPT_SAVE_DIR="/data/cache"
14-DATA_PATH="/data/datasets/internlm3/enwiki_text_document"14+DATA_PATH="/data/ci/datasets/processed/internlm3/enwiki_text_document"
15TOKENIZER_MODEL="/data/hf/internlm3-hf/tokenizer.model"15TOKENIZER_MODEL="/data/hf/internlm3-hf/tokenizer.model"
16CKPT_LOAD_DIR="/data/pipeline/internlm3_tp1_pp4"16CKPT_LOAD_DIR="/data/pipeline/internlm3_tp1_pp4"
17 17 
Mtests/pipeline/llama2/llama2_tp1_pp8_patch_gloo_ptd.sh+3-3
@@ -15,9 +15,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
15basepath=$(cd `dirname $0`; cd ../../../; pwd)15basepath=$(cd `dirname $0`; cd ../../../; pwd)
16 16 
17 17 
18-DATA_PATH="/data/datasets/llama2_7b_pretrain/alpaca_text_document"18+DATA_PATH="/data/ci/datasets/processed/llama2_7b_pretrain/alpaca_text_document"
19-TOKENIZER_MODEL="/data/hf/llama-2-7b-hf/tokenizer.model"19+TOKENIZER_MODEL="/data/ci/models/llama2/hf/llama-2-7b-hf/tokenizer.model"
20-CKPT_LOAD_DIR="/data/pipeline/llama-2-7b-mcore-tp1-pp8"20+CKPT_LOAD_DIR="/data/ci/models/llama2/mg/llama-2-7b-mcore-tp1-pp8"
21 21 
22TP=122TP=1
23PP=823PP=8
Mtests/pipeline/llama2/llama2_tp4pp2vpp2_tp2d_tpx2tpy2.sh+3-3
@@ -13,9 +13,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
13basepath=$(cd `dirname $0`; cd ../../../; pwd)13basepath=$(cd `dirname $0`; cd ../../../; pwd)
14 14 
15 15 
16-DATA_PATH="/data/pretrain_dataset/alpaca_text_document"16+DATA_PATH="/data/ci/datasets/processed/pretrain_dataset/alpaca_text_document"
17-TOKENIZER_MODEL="/data/hf/llama-2-7b-hf/tokenizer.model"17+TOKENIZER_MODEL="/data/ci/models/llama2/hf/llama-2-7b-hf/tokenizer.model"
18-CKPT_LOAD_DIR="/data/llama2-2dtp-tp4pp2vp2tpx2tpy2"18+CKPT_LOAD_DIR="/data/ci/models/llama2/mg/llama2-2dtp-tp4pp2vp2tpx2tpy2"
19 19 
20TP=420TP=4
21PP=221PP=2
Mtests/pipeline/llama2/llama2_tp8_pp1_coc_ptd.sh+3-3
@@ -14,9 +14,9 @@ basepath=$(cd `dirname $0`; cd ../../../; pwd)
14 14 
15 15 
16CKPT_SAVE_DIR="./ckpt"16CKPT_SAVE_DIR="./ckpt"
17-DATA_PATH="/data/datasets/llama2_7b_pretrain/alpaca_text_document"17+DATA_PATH="/data/ci/datasets/processed/llama2_7b_pretrain/alpaca_text_document"
18-TOKENIZER_MODEL="/data/hf/llama-2-7b-hf/tokenizer.model"18+TOKENIZER_MODEL="/data/ci/models/llama2/hf/llama-2-7b-hf/tokenizer.model"
19-CKPT_LOAD_DIR="/data/pipeline/llama2-7b-tp8-pp1-coc"19+CKPT_LOAD_DIR="/data/ci/models/llama2/mg/llama2-7b-tp8-pp1-coc"
20 20 
21TP=821TP=8
22PP=122PP=1
Mtests/pipeline/llama2/tune_llama2_tp1_pp1_lora_ptd.sh+3-3
@@ -15,9 +15,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
15basepath=$(cd `dirname $0`; cd ../../../; pwd)15basepath=$(cd `dirname $0`; cd ../../../; pwd)
16 16 
17CKPT_SAVE_DIR="/data/cache"17CKPT_SAVE_DIR="/data/cache"
18-CKPT_LOAD_DIR="/data/pipeline/llama-2-7b-mcore-tp1pp1"18+CKPT_LOAD_DIR="/data/ci/models/llama2/mg/llama-2-7b-mcore-tp1pp1"
19-DATA_PATH="/data/datasets/llama2_7b_finetune/alpaca"19+DATA_PATH="/data/ci/datasets/processed/llama2_7b_finetune/alpaca"
20-TOKENIZER_MODEL="/data/hf/llama-2-7b-hf "20+TOKENIZER_MODEL="/data/ci/models/llama2/hf/llama-2-7b-hf "
21 21 
22TP=122TP=1
23PP=123PP=1
Mtests/pipeline/llama2/tune_llama2_tp1_pp1_lu_lora_ptd.sh+3-3
@@ -15,9 +15,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
15basepath=$(cd `dirname $0`; cd ../../../; pwd)15basepath=$(cd `dirname $0`; cd ../../../; pwd)
16 16 
17CKPT_SAVE_DIR="/data/cache"17CKPT_SAVE_DIR="/data/cache"
18-CKPT_LOAD_DIR="/data/pipeline/llama-2-7b-mcore-tp1pp1"18+CKPT_LOAD_DIR="/data/ci/models/llama2/mg/llama-2-7b-mcore-tp1pp1"
19-DATA_PATH="/data/datasets/llama2_7b_finetune/alpaca"19+DATA_PATH="/data/ci/datasets/processed/llama2_7b_finetune/alpaca"
20-TOKENIZER_MODEL="/data/hf/llama-2-7b-hf "20+TOKENIZER_MODEL="/data/ci/models/llama2/hf/llama-2-7b-hf "
21 21 
22TP=122TP=1
23PP=123PP=1
Mtests/pipeline/llama2/tune_llama2_tp2_pp4_vpp2_mcore_full.sh+4-4
@@ -10,10 +10,10 @@ NNODES=1
10NODE_RANK=010NODE_RANK=0
11WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))11WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
12 12 
13-DATA_PATH="/data/datasets/llama2_finetune/alpaca"13+DATA_PATH="/data/ci/datasets/processed/llama2_finetune/alpaca"
14-TOKENIZER_MODEL="/data/hf/llama-2-7b-hf/tokenizer.model"14+TOKENIZER_MODEL="/data/ci/models/llama2/hf/llama-2-7b-hf/tokenizer.model"
15-CKPT_LOAD_DIR="/data/pipeline/llama-2-7b-mg-tp2-pp4-mcore-vpp2/"15+CKPT_LOAD_DIR="/data/ci/models/llama2/mg/llama-2-7b-mg-tp2-pp4-mcore-vpp2/"
16-TOKENIZER_PATH="/data/hf/llama-2-7b-hf/"16+TOKENIZER_PATH="/data/ci/models/llama2/hf/llama-2-7b-hf/"
17 17 
18basepath=$(cd `dirname $0`; cd ../../../; pwd)18basepath=$(cd `dirname $0`; cd ../../../; pwd)
19 19 
Mtests/pipeline/llama3/llama3_tp2_pp2_vpp1.sh+3-3
@@ -10,9 +10,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
10 10 
11basepath=$(cd `dirname $0`; cd ../../../; pwd)11basepath=$(cd `dirname $0`; cd ../../../; pwd)
12 12 
13-CKPT_LOAD_DIR="/data/llama3-8b-tp2-pp2-vpp1"13+CKPT_LOAD_DIR="/data/ci/models/llama3/mg/llama3-8b-tp2-pp2-vpp1"
14-DATA_PATH="/data/llama-3-8b-dataset/alpaca_text_document"14+DATA_PATH="/data/ci/datasets/processed/llama-3-8b-dataset/alpaca_text_document"
15-TOKENIZER_PATH="/data/llama-3-8b-hf/"15+TOKENIZER_PATH="/data/ci/models/llama3/hf/llama-3-8b-hf/"
16 16 
17TP=217TP=2
18PP=218PP=2
Mtests/pipeline/mamba2/mamba2_2.7b_tp1_pp1.sh+3-3
@@ -22,9 +22,9 @@ DISTRIBUTED_ARGS="
22 22 
23echo "NODE_RANK ${NODE_RANK}"23echo "NODE_RANK ${NODE_RANK}"
24 24 
25-DATA_PATH="/data/mamba2_2.7b_enwiki/enwiki_50288_text_document"25+DATA_PATH="/data/ci/datasets/processed/mamba2_2.7b_enwiki/enwiki_50288_text_document"
26-CKPT_LOAD_DIR="/data/mamba2_2.7b_tp1pp1_ngroup1"26+CKPT_LOAD_DIR="/data/ci/models/mamba2/mg/mamba2_2.7b_tp1pp1_ngroup1"
27-TOKENIZER_PATH="/data/mamba2-hf/mamba2_2.7b_from_8b.model"27+TOKENIZER_PATH="/data/ci/models/mamba2/hf/mamba2-hf/mamba2_2.7b_from_8b.model"
28 28 
29TP=129TP=1
30PP=130PP=1
Mtests/pipeline/minicpm/minicpm_2b_tp1_pp1.sh+2-2
@@ -11,8 +11,8 @@ NODE_RANK=0
11WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))11WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
12 12 
13CKPT_SAVE_DIR="/data/cache"13CKPT_SAVE_DIR="/data/cache"
14-DATA_PATH="/data/datasets/minicpm/enwiki_text_document"14+DATA_PATH="/data/ci/datasets/processed/minicpm/enwiki_text_document"
15-TOKENIZER_MODEL="/data/hf/minicpm_hf/tokenizer.model"15+TOKENIZER_MODEL="/data/ci/models/minicpm/hf/minicpm_hf/tokenizer.model"
16CKPT_LOAD_DIR="/data/pipeline/minicpm_tp1pp1"16CKPT_LOAD_DIR="/data/pipeline/minicpm_tp1pp1"
17 17 
18TP=118TP=1
Mtests/pipeline/mixtral/mixtral_tp1_pp4_ep2_drop_mcore.sh+3-3
@@ -21,9 +21,9 @@ DISTRIBUTED_ARGS="
21echo "NODE_RANK ${NODE_RANK}"21echo "NODE_RANK ${NODE_RANK}"
22 22 
23 23 
24-DATA_PATH="/data/datasets/Mixtral-8x7B_dataset/alpaca_text_document"24+DATA_PATH="/data/ci/datasets/processed/Mixtral-8x7B_dataset/alpaca_text_document"
25-TOKENIZER_MODEL="/data/hf/mixtral_moe_tp1pp4_hf/tokenizer.model"25+TOKENIZER_MODEL="/data/ci/models/mixtral/hf/mixtral_moe_tp1pp4_hf/tokenizer.model"
26-CKPT_LOAD_DIR="/data/pipeline/mixtral_moe_tp1pp4_ckpt/"26+CKPT_LOAD_DIR="/data/ci/models/mixtral/mg/mixtral_moe_tp1pp4_ckpt/"
27CKPT_SAVE_DIR="/data/cache"27CKPT_SAVE_DIR="/data/cache"
28 28 
29TP=129TP=1
Mtests/pipeline/mixtral/tune_mixtral_tp2_pp2_lora_ptd.sh+3-3
@@ -18,9 +18,9 @@ DISTRIBUTED_ARGS="
18 --master_port $MASTER_PORT18 --master_port $MASTER_PORT
19"19"
20 20 
21-DATA_PATH="/data/datasets/Mixtral-8x7B_Finetune/alpaca"21+DATA_PATH="/data/ci/datasets/processed/Mixtral-8x7B_Finetune/alpaca"
22-TOKENIZER_MODEL="/data/hf/Mixtral-8x7B-Hf/"22+TOKENIZER_MODEL="/data/ci/models/mixtral/hf/Mixtral-8x7B-Hf/"
23-CKPT_LOAD_DIR="/data/pipeline/Mixtral-tp2-pp2-ep1-mcore"23+CKPT_LOAD_DIR="/data/ci/models/mixtral/mg/Mixtral-tp2-pp2-ep1-mcore"
24 24 
25basepath=$(cd `dirname $0`; cd ../../../; pwd)25basepath=$(cd `dirname $0`; cd ../../../; pwd)
26 26 
Mtests/pipeline/phi35-moe/param_config.json+10-10
@@ -7,11 +7,11 @@
7 "moe-grouped-gemm": null,7 "moe-grouped-gemm": null,
8 "spec": ["mindspeed_llm.tasks.models.spec.phi35_moe_spec", "layer_spec"],8 "spec": ["mindspeed_llm.tasks.models.spec.phi35_moe_spec", "layer_spec"],
9 "task": "mmlu",9 "task": "mmlu",
10- "task-data-path": "/data/eval_data/mmlu/data/test",10+ "task-data-path": "/data/ci/datasets/eval_data/mmlu/data/test",
11 "tensor-model-parallel-size": 1,11 "tensor-model-parallel-size": 1,
12 "pipeline-model-parallel-size": 8,12 "pipeline-model-parallel-size": 8,
13 "tokenizer-type": "PretrainedFromHF",13 "tokenizer-type": "PretrainedFromHF",
14- "tokenizer-name-or-path": "/data/hf/Phi-3.5-MoE-instruct-hf",14+ "tokenizer-name-or-path": "/data/ci/models/phi35/hf/Phi-3.5-MoE-instruct-hf",
15 "seq-length": 4096,15 "seq-length": 4096,
16 "max-position-embeddings": 131072,16 "max-position-embeddings": 131072,
17 "num-layers": 32,17 "num-layers": 32,
@@ -26,7 +26,7 @@
26 "moe-input-jitter-eps": 0.01,26 "moe-input-jitter-eps": 0.01,
27 "moe-permutation-async-comm": null,27 "moe-permutation-async-comm": null,
28 "swiglu": null,28 "swiglu": null,
29- "load": "/data/pipeline/phi35-moe-tp1pp8-mcore-base/",29+ "load": "/data/ci/models/phi35/mg/phi35-moe-tp1pp8-mcore-base/",
30 "evaluation-batch-size": 1,30 "evaluation-batch-size": 1,
31 "micro-batch-size": 1,31 "micro-batch-size": 1,
32 "max-new-tokens": 1,32 "max-new-tokens": 1,
@@ -71,7 +71,7 @@
71 "tensor-model-parallel-size": 1,71 "tensor-model-parallel-size": 1,
72 "pipeline-model-parallel-size": 8,72 "pipeline-model-parallel-size": 8,
73 "tokenizer-type": "PretrainedFromHF",73 "tokenizer-type": "PretrainedFromHF",
74- "tokenizer-name-or-path": "/data/hf/Phi-3.5-MoE-instruct-hf",74+ "tokenizer-name-or-path": "/data/ci/models/phi35/hf/Phi-3.5-MoE-instruct-hf",
75 "seq-length": 4096,75 "seq-length": 4096,
76 "max-position-embeddings": 131072,76 "max-position-embeddings": 131072,
77 "num-layers": 32,77 "num-layers": 32,
@@ -86,7 +86,7 @@
86 "moe-input-jitter-eps": 0.01,86 "moe-input-jitter-eps": 0.01,
87 "moe-permutation-async-comm": null,87 "moe-permutation-async-comm": null,
88 "swiglu": null,88 "swiglu": null,
89- "load": "/data/pipeline/phi35-moe-tp1pp8-mcore-base",89+ "load": "/data/ci/models/phi35/mg/phi35-moe-tp1pp8-mcore-base",
90 "micro-batch-size": 1,90 "micro-batch-size": 1,
91 "max-new-tokens": 256,91 "max-new-tokens": 256,
92 "padded-vocab-size": 32064,92 "padded-vocab-size": 32064,
@@ -123,9 +123,9 @@
123 "test_preprocess_pretrain_data": [123 "test_preprocess_pretrain_data": [
124 {124 {
125 "param": {125 "param": {
126- "input": "/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet",126+ "input": "/data/ci/datasets/origin/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
127 "tokenizer-type": "PretrainedFromHF",127 "tokenizer-type": "PretrainedFromHF",
128- "tokenizer-name-or-path": "/data/hf/Phi-3.5-MoE-instruct-hf"128+ "tokenizer-name-or-path": "/data/ci/models/phi35/hf/Phi-3.5-MoE-instruct-hf"
129 },129 },
130 "prefix": "alpaca"130 "prefix": "alpaca"
131 }131 }
@@ -141,9 +141,9 @@
141 "save-model-type": "mg",141 "save-model-type": "mg",
142 "target-tensor-parallel-size": "1",142 "target-tensor-parallel-size": "1",
143 "target-pipeline-parallel-size": "8",143 "target-pipeline-parallel-size": "8",
144- "load-dir": "/data/hf/Phi-3.5-MoE-instruct-hf",144+ "load-dir": "/data/ci/models/phi35/hf/Phi-3.5-MoE-instruct-hf",
145- "save-dir": "/data/cache/test_phi35_moe_hf2mg_tp1pp8_save",145+ "save-dir": "/data/ci/cache/test_phi35_moe_hf2mg_tp1pp8_save",
146- "tokenizer-model": "/data/hf/Phi-3.5-MoE-instruct-hf/tokenizer.model",146+ "tokenizer-model": "/data/ci/models/phi35/hf/Phi-3.5-MoE-instruct-hf/tokenizer.model",
147 "spec": "mindspeed_llm.tasks.models.spec.phi35_moe_spec layer_spec",147 "spec": "mindspeed_llm.tasks.models.spec.phi35_moe_spec layer_spec",
148 "params-dtype": "bf16",148 "params-dtype": "bf16",
149 "add-qkv-bias": null,149 "add-qkv-bias": null,
Mtests/pipeline/phi35-moe/phi35_moe_tp1_pp8_mcore.sh+3-3
@@ -11,9 +11,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
11 11 
12basepath=$(cd `dirname $0`; cd ../../../; pwd)12basepath=$(cd `dirname $0`; cd ../../../; pwd)
13 13 
14-CKPT_LOAD_DIR="/data/pipeline/phi35-moe-tp1pp8l8-mcore-base"14+CKPT_LOAD_DIR="/data/ci/models/phi35/mg/phi35-moe-tp1pp8l8-mcore-base"
15-DATA_PATH="/data/datasets/phi35-moe-data/alpaca_text_document"15+DATA_PATH="/data/ci/datasets/processed/phi35-moe-data/alpaca_text_document"
16-TOKENIZER_MODEL="/data/hf/Phi-3.5-MoE-instruct-hf"16+TOKENIZER_MODEL="/data/ci/models/phi35/hf/Phi-3.5-MoE-instruct-hf"
17 17 
18TP=118TP=1
19PP=819PP=8
Mtests/pipeline/qwen15-7B/param_config.json+8-8
@@ -2,9 +2,9 @@
2 "test_preprocess_pretrain_data": [2 "test_preprocess_pretrain_data": [
3 {3 {
4 "param": {4 "param": {
5- "input": "/data/train-00000-of-00001-a09b74b3ef9c3b56.parquet",5+ "input": "/data/ci/datasets/origin/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
6 "tokenizer-type": "PretrainedFromHF",6 "tokenizer-type": "PretrainedFromHF",
7- "tokenizer-name-or-path": "/data/hf/qwen15-7B-hf/"7+ "tokenizer-name-or-path": "/data/ci/models/qwen15-7b/hf/qwen15-7B-hf/"
8 },8 },
9 "prefix": "alpaca"9 "prefix": "alpaca"
10 }10 }
@@ -13,7 +13,7 @@
13 "test_qwen15_7B_mmlu_ppl_evaluate": [13 "test_qwen15_7B_mmlu_ppl_evaluate": [
14 {14 {
15 "param": {15 "param": {
16- "task-data-path": "/data/eval_data/mmlu/data/test/",16+ "task-data-path": "/data/ci/datasets/eval_data/mmlu/data/test/",
17 "task": "mmlu_ppl",17 "task": "mmlu_ppl",
18 "use-mcore-models": null,18 "use-mcore-models": null,
19 "tensor-model-parallel-size": 8,19 "tensor-model-parallel-size": 8,
@@ -27,10 +27,10 @@
27 "disable-bias-linear": null,27 "disable-bias-linear": null,
28 "swiglu": null,28 "swiglu": null,
29 "position-embedding-type": "rope",29 "position-embedding-type": "rope",
30- "load": "/data/hf/qwen15-7B-hf/ckpt-mcore/qwen15_7b_tp8_pp1/",30+ "load": "/data/ci/models/qwen15-7b/mg/qwen15_7b_tp8_pp1",
31 "normalization": "RMSNorm",31 "normalization": "RMSNorm",
32 "tokenizer-type": "PretrainedFromHF",32 "tokenizer-type": "PretrainedFromHF",
33- "tokenizer-name-or-path": "/data/hf/qwen15-7B-hf/",33+ "tokenizer-name-or-path": "/data/ci/models/qwen15-7b/hf/qwen15-7B-hf/",
34 "tokenizer-not-use-fast": null,34 "tokenizer-not-use-fast": null,
35 "micro-batch-size": 1,35 "micro-batch-size": 1,
36 "no-load-rng": null,36 "no-load-rng": null,
@@ -54,7 +54,7 @@
54 "test_qwen15_7B_human_eval_evaluate": [54 "test_qwen15_7B_human_eval_evaluate": [
55 {55 {
56 "param": {56 "param": {
57- "task-data-path": "/data/pipeline/eval_data/human_eval_alternative/",57+ "task-data-path": "/data/ci/datasets/eval_data/human_eval_alternative/",
58 "task": "human_eval",58 "task": "human_eval",
59 "use-mcore-models": null,59 "use-mcore-models": null,
60 "tensor-model-parallel-size": 8,60 "tensor-model-parallel-size": 8,
@@ -68,10 +68,10 @@
68 "disable-bias-linear": null,68 "disable-bias-linear": null,
69 "swiglu": null,69 "swiglu": null,
70 "position-embedding-type": "rope",70 "position-embedding-type": "rope",
71- "load": "/data/hf/qwen15-7B-hf/ckpt-mcore/qwen15_7b_tp8_pp1/",71+ "load": "/data/ci/models/qwen15-7b/mg/qwen15_7b_tp8_pp1",
72 "normalization": "RMSNorm",72 "normalization": "RMSNorm",
73 "tokenizer-type": "PretrainedFromHF",73 "tokenizer-type": "PretrainedFromHF",
74- "tokenizer-name-or-path": "/data/hf/qwen15-7B-hf/",74+ "tokenizer-name-or-path": "/data/ci/models/qwen15-7b/hf/qwen15-7B-hf/",
75 "tokenizer-not-use-fast": null,75 "tokenizer-not-use-fast": null,
76 "micro-batch-size": 1,76 "micro-batch-size": 1,
77 "no-load-rng": null,77 "no-load-rng": null,
Mtests/pipeline/qwen25/qwen2_moe_tp1_pp2_ep2_cp2_32k.sh+3-3
@@ -14,9 +14,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
14basepath=$(cd `dirname $0`; cd ../../../; pwd)14basepath=$(cd `dirname $0`; cd ../../../; pwd)
15 15 
16# please fill these path configurations16# please fill these path configurations
17-CKPT_LOAD_DIR="/data/qwen2-moe-t1p2e2-new-spec"17+CKPT_LOAD_DIR="/data/ci/models/qwen2-moe/mg/qwen2-moe-t1p2e2-new-spec"
18-DATA_PATH="/data/qwen2_moe_dataset/enwiki_text_document"18+DATA_PATH="/data/ci/datasets/processed/qwen2_moe_dataset/enwiki_text_document"
19-TOKENIZER_PATH="/data/qwen2-moe-hf"19+TOKENIZER_PATH="/data/ci/models/qwen2-moe/hf/qwen2-moe-hf"
20 20 
21TP=121TP=1
22PP=222PP=2
Mtests/pipeline/qwen25/tune_qwen25_0point5b_tp1_pp1_pack.sh+2-2
@@ -10,8 +10,8 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
10 10 
11# please fill these path configurations11# please fill these path configurations
12CKPT_SAVE_DIR="/data/cache"12CKPT_SAVE_DIR="/data/cache"
13-DATA_PATH="/data/datasets/qwen25_finetune/alpaca"13+DATA_PATH="/data/ci/datasets/processed/qwen25_finetune/alpaca"
14-TOKENIZER_PATH="/data/hf/qwen25-5b-hf"14+TOKENIZER_PATH="/data/ci/models/qwen25/hf/qwen25-5b-hf"
15CKPT_LOAD_DIR="/data/pipeline/qwen25_tp1pp1"15CKPT_LOAD_DIR="/data/pipeline/qwen25_tp1pp1"
16 16 
17TP=117TP=1
Mtests/rlxf/dpo_full_llama3_8b_ptd_tp2pp2vpp2cp2.sh+1-1
@@ -11,7 +11,7 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
11 11 
12basepath=$(cd `dirname $0`; cd ../../../; pwd)12basepath=$(cd `dirname $0`; cd ../../../; pwd)
13 13 
14-DATA_PATH="/data/pairwise_dataset/baseline/orca_rlhf/orca_rlhf_llama3"14+DATA_PATH="/data/ci/datasets/origin/pairwise_dataset/baseline/orca_rlhf/orca_rlhf_llama3"
15TOKENIZER_MODEL="/data/Meta-Llama-3-8B-Instruct"15TOKENIZER_MODEL="/data/Meta-Llama-3-8B-Instruct"
16CKPT_LOAD_DIR="/data/llama-3-8b_tp2pp2vpp2"16CKPT_LOAD_DIR="/data/llama-3-8b_tp2pp2vpp2"
17 17 
Mtests/rlxf/dpo_full_mixtral_8x7b_ptd_tp1pp2vpp2ep2cp2.sh+1-1
@@ -23,7 +23,7 @@ DISTRIBUTED_ARGS="
23 23 
24echo "NODE_RANK ${NODE_RANK}"24echo "NODE_RANK ${NODE_RANK}"
25 25 
26-DATA_PATH="/data/pairwise_dataset/baseline/orca_rlhf/orca_rlhf_mixtral"26+DATA_PATH="/data/ci/datasets/origin/pairwise_dataset/baseline/orca_rlhf/orca_rlhf_mixtral"
27TOKENIZER_MODEL="/data/Mixtral-8x7B-v0.1/"27TOKENIZER_MODEL="/data/Mixtral-8x7B-v0.1/"
28CKPT_LOAD_DIR="/data/mixtral_8x7b_tp1pp2vpp2ep2/"28CKPT_LOAD_DIR="/data/mixtral_8x7b_tp1pp2vpp2ep2/"
29 29 
Mtests/rlxf/dpo_lora_mixtral_8x7b_ptd_tp2pp1ep2cp2.sh+1-1
@@ -23,7 +23,7 @@ DISTRIBUTED_ARGS="
23 23 
24echo "NODE_RANK ${NODE_RANK}"24echo "NODE_RANK ${NODE_RANK}"
25 25 
26-DATA_PATH="/data/pairwise_dataset/baseline/orca_rlhf/orca_rlhf_mixtral"26+DATA_PATH="/data/ci/datasets/origin/pairwise_dataset/baseline/orca_rlhf/orca_rlhf_mixtral"
27TOKENIZER_MODEL="/data/Mixtral-8x7B-v0.1/"27TOKENIZER_MODEL="/data/Mixtral-8x7B-v0.1/"
28CKPT_LOAD_DIR="/data/mixtral_8x7b_tp2pp1ep2/"28CKPT_LOAD_DIR="/data/mixtral_8x7b_tp2pp1ep2/"
29LORA_CKPT_DIR='./tmp'29LORA_CKPT_DIR='./tmp'
Mtests/rlxf/train_orm_llama2_7b_pp2_vpp2_dp2.sh+1-1
@@ -18,7 +18,7 @@ DISTRIBUTED_ARGS="
18 18 
19DATA_PATH="/data/ci/orm/dpo-en-llama-2-7b/dpo_en"19DATA_PATH="/data/ci/orm/dpo-en-llama-2-7b/dpo_en"
20TOKENIZER_PATH="/data/hf/llama-2-7b-hf/"20TOKENIZER_PATH="/data/hf/llama-2-7b-hf/"
21-CKPT_LOAD_DIR="/data/ci/orm/llama-2-7b-layers8-rm-mcore_pp2vpp2/"21+CKPT_LOAD_DIR="/data/ci/models/llama2/mg/llama-2-7b-layers8-rm-mcore_pp2vpp2/"
22 22 
23basepath=$(cd `dirname $0`; cd ../../../; pwd)23basepath=$(cd `dirname $0`; cd ../../../; pwd)
24 24 
Mtests/st/shell_scripts/chatglm3_gqa_cp4.sh+3-3
@@ -10,9 +10,9 @@ WORLD_SIZE=$((NPUS_PER_NODE*$NNODES))
10 10 
11basepath=$(cd `dirname $0`; cd ../../../; pwd)11basepath=$(cd `dirname $0`; cd ../../../; pwd)
12 12 
13-DATA_PATH="/data/datasets/chatglm3-dataset-alpaca/alpaca_text_document"13+DATA_PATH="/data/ci/datasets/processed/chatglm3-dataset-alpaca/alpaca_text_document"
14-TOKENIZER_PATH="/data/hf/chatglm3-6b-base-hf/"14+TOKENIZER_PATH="/data/ci/models/chatglm3/hf/chatglm3-6b-base-hf/"
15-CKPT_LOAD_DIR="/data/chatglm3-6b-tp1-pp1-cp8/"15+CKPT_LOAD_DIR="/data/ci/models/chatglm3/mg/chatglm3-6b-tp1-pp1-cp8/"
16 16 
17TP=117TP=1
18PP=118PP=1
Mtests/st/shell_scripts/deepseek_500b_tp1_pp2_ep2_cp2_overlap.sh+3-3
@@ -11,9 +11,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
11 11 
12basepath=$(cd `dirname $0`; cd ../../../; pwd)12basepath=$(cd `dirname $0`; cd ../../../; pwd)
13 13 
14-CKPT_LOAD_DIR="/data/deepseek-500b-tp1-pp2-ep2-cp2-overlap-base/"14+CKPT_LOAD_DIR="/data/ci/models/deepseek-v2/mg/deepseek-500b-tp1-pp2-ep2-cp2-overlap-base/"
15-DATA_PATH="/data/pretrain_dataset/alpaca_text_document"15+DATA_PATH="/data/ci/datasets/processed/pretrain_dataset/alpaca_text_document"
16-TOKENIZER_MODEL="/data/hf/llama-2-7b-hf"16+TOKENIZER_MODEL="/data/ci/models/llama2/hf/llama-2-7b-hf"
17 17 
18TP=118TP=1
19PP=219PP=2
Mtests/st/shell_scripts/deepseek_v2_mcore_tp1_pp1_ep8.sh+2-2
@@ -11,9 +11,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
11 11 
12basepath=$(cd `dirname $0`; cd ../../../; pwd)12basepath=$(cd `dirname $0`; cd ../../../; pwd)
13 13 
14-DATA_PATH="/data/pretrain_dataset/alpaca_text_document"14+DATA_PATH="/data/ci/datasets/processed/pretrain_dataset/alpaca_text_document"
15TOKENIZER_MODEL="/data/gemm/"15TOKENIZER_MODEL="/data/gemm/"
16-CKPT_LOAD_DIR=" /data/ci/deepseek2/mg_base/l2_gemm_t1p1e8_new/"16+CKPT_LOAD_DIR=" /data/ci/models/deepseek2/mg/l2_gemm_t1p1e8_new/"
17 17 
18TP=118TP=1
19PP=119PP=1
Mtests/st/shell_scripts/deepseek_v3_mcore_tp1_pp2_ep4.sh+3-3
@@ -12,9 +12,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
12 12 
13basepath=$(cd `dirname $0`; cd ../../../; pwd)13basepath=$(cd `dirname $0`; cd ../../../; pwd)
14 14 
15-DATA_PATH=/data/pretrain_dataset/alpaca_text_document15+DATA_PATH=/data/ci/datasets/processed/pretrain_dataset/alpaca_text_document
16-TOKENIZER_PATH=/data/deepseek-v3-mcore-tp1-pp2-ep4-16experts16+TOKENIZER_PATH=/data/ci/models/deepseek3/mg/deepseek-v3-mcore-tp1-pp2-ep4-16experts
17-CKPT_LOAD_DIR=/data/ci/deepseek2/mg_base/deepseek-v3-mcore-tp1-pp2-ep4-16expert17+CKPT_LOAD_DIR=/data/ci/models/deepseek2/mg/deepseek-v3-mcore-tp1-pp2-ep4-16expert
18 18 
19TP=119TP=1
20PP=220PP=2
Mtests/st/shell_scripts/deepseek_v3_mcore_tp2_pp2_ep2_dualpipev_fb.sh+3-3
@@ -14,9 +14,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
14 14 
15basepath=$(cd `dirname $0`; cd ../../../; pwd)15basepath=$(cd `dirname $0`; cd ../../../; pwd)
16 16 
17-DATA_PATH=/data/datasets/alpaca_dsv3_eod/alpaca_text_document17+DATA_PATH=/data/ci/datasets/processed/alpaca_dsv3_eod/alpaca_text_document
18-TOKENIZER_PATH=/data/deepseek-v3-l4-tp2pp2ep4-dualpipev/18+TOKENIZER_PATH=/data/ci/models/deepseek-500b/mg/deepseek-v3-l4-tp2pp2ep4-dualpipev/
19-CKPT_LOAD_DIR=/data/deepseek-v3-l4-tp2pp2ep4-dualpipev/19+CKPT_LOAD_DIR=/data/ci/models/deepseek-500b/mg/deepseek-v3-l4-tp2pp2ep4-dualpipev/
20 20 
21TP=221TP=2
22PP=222PP=2
Mtests/st/shell_scripts/llama2_tp2_cp4_general_double_ring.sh+3-3
@@ -13,9 +13,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
13 13 
14basepath=$(cd `dirname $0`; cd ../../../; pwd)14basepath=$(cd `dirname $0`; cd ../../../; pwd)
15 15 
16-CKPT_LOAD_DIR="/data/ci/Llama2-mcore-tp2/"16+CKPT_LOAD_DIR="/data/ci/models/llama2/mg/Llama2-mcore-tp2/"
17-DATA_PATH="/data/ci/alpaca_llama2_eod/alpaca_llama2_text_document"17+DATA_PATH="/data/ci/datasets/processed/alpaca_llama2_eod/alpaca_llama2_text_document"
18-TOKENIZER_MODEL="/data/hf/llama-2-7b-hf/tokenizer.model"18+TOKENIZER_MODEL="/data/ci/models/llama2/hf/llama-2-7b-hf/tokenizer.model"
19 19 
20TP=220TP=2
21PP=121PP=1
Mtests/st/shell_scripts/llama2_tp2_pp4_vpp2_ptd.sh+3-3
@@ -13,9 +13,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
13 13 
14basepath=$(cd `dirname $0`; cd ../../../; pwd)14basepath=$(cd `dirname $0`; cd ../../../; pwd)
15 15 
16-CKPT_LOAD_DIR="/data/ci/llama-2-7b-mg-tp2-pp4-mcore-vpp2-test"16+CKPT_LOAD_DIR="/data/ci/models/llama2/mg/llama-2-7b-mg-tp2-pp4-mcore-vpp2-test"
17-DATA_PATH="/data/ci/alpaca_llama2_eod/alpaca_llama2_text_document"17+DATA_PATH="/data/ci/datasets/processed/alpaca_llama2_eod/alpaca_llama2_text_document"
18-TOKENIZER_MODEL="/data/hf/llama-2-7b-hf/tokenizer.model"18+TOKENIZER_MODEL="/data/ci/models/llama2/hf/llama-2-7b-hf/tokenizer.model"
19 19 
20TP=220TP=2
21PP=421PP=4
Mtests/st/shell_scripts/llama2_tp2_pp4_vpp2_swap.sh+3-3
@@ -15,9 +15,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
15 15 
16basepath=$(cd `dirname $0`; cd ../../../; pwd)16basepath=$(cd `dirname $0`; cd ../../../; pwd)
17 17 
18-CKPT_LOAD_DIR="/data/ci/llama-2-7b-mg-tp2-pp4-mcore-vpp2-test"18+CKPT_LOAD_DIR="/data/ci/models/llama2/mg/llama-2-7b-mg-tp2-pp4-mcore-vpp2-test"
19-DATA_PATH="/data/pretrain_dataset/alpaca_text_document"19+DATA_PATH="/data/ci/datasets/processed/pretrain_dataset/alpaca_text_document"
20-TOKENIZER_MODEL="/data/hf/llama-2-7b-hf/tokenizer.model"20+TOKENIZER_MODEL="/data/ci/models/llama2/hf/llama-2-7b-hf/tokenizer.model"
21 21 
22TP=222TP=2
23PP=423PP=4
Mtests/st/shell_scripts/llama3_mcore_tp2_pp2_vpp2_noop_layer.sh+3-3
@@ -10,9 +10,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
10 10 
11basepath=$(cd `dirname $0`; cd ../../../; pwd)11basepath=$(cd `dirname $0`; cd ../../../; pwd)
12 12 
13-DATA_PATH="/data/llama-3-8b-dataset/alpaca_text_document"13+DATA_PATH="/data/ci/datasets/processed/llama-3-8b-dataset/alpaca_text_document"
14-TOKENIZER_MODEL="/data/llama-3-8b-hf/"14+TOKENIZER_MODEL="/data/ci/models/llama3/hf/llama-3-8b-hf/"
15-CKPT_LOAD_DIR="/data/llama-3-8b-hf-nooplayer-tp2pp2vpp2-mcore-base/"15+CKPT_LOAD_DIR="/data/ci/models/llama3/mg/llama-3-8b-hf-nooplayer-tp2pp2vpp2-mcore-base/"
16TP=216TP=2
17PP=217PP=2
18 18 
Mtests/st/shell_scripts/mamba2_8b_tp8_pp1_4k_ptd.sh+3-3
@@ -22,9 +22,9 @@ DISTRIBUTED_ARGS="
22 22 
23echo "NODE_RANK ${NODE_RANK}"23echo "NODE_RANK ${NODE_RANK}"
24 24 
25-DATA_PATH="/data/mamba_enwiki/mamba_enwiki_text_document"25+DATA_PATH="/data/ci/datasets/processed/mamba_enwiki/mamba_enwiki_text_document"
26-CKPT_LOAD_DIR="/data/mamba2_tp8dp1_mbs2_gbs8"26+CKPT_LOAD_DIR="/data/ci/models/mamba2/mg/mamba2_tp8dp1_mbs2_gbs8"
27-TOKENIZER_PATH="/data/mamba-hf/mt_nlg_plus_multilingual_ja_zh_the_stack_frac_015_256k.model"27+TOKENIZER_PATH="/data/ci/models/mamba/hf/mamba-hf/mt_nlg_plus_multilingual_ja_zh_the_stack_frac_015_256k.model"
28 28 
29 29 
30 30 
Mtests/st/shell_scripts/mixtral_mcore_tp4_cp2_ep2_ptd.sh+3-3
@@ -20,9 +20,9 @@ DISTRIBUTED_ARGS="
20 20 
21echo "NODE_RANK ${NODE_RANK}"21echo "NODE_RANK ${NODE_RANK}"
22 22 
23-DATA_PATH="/data/pretrain_dataset/alpaca_text_document"23+DATA_PATH="/data/ci/datasets/processed/pretrain_dataset/alpaca_text_document"
24-TOKENIZER_MODEL="/data/mixtral-8-7b-hf/Mixtral-8x7B/tokenizer.model"24+TOKENIZER_MODEL="/data/ci/models/mixtral/hf/Mixtral-8x7B/tokenizer.model"
25-CKPT_LOAD_DIR="/data/mixtral-8*7b-tp4-ep2-mcore-base"25+CKPT_LOAD_DIR="/data/ci/models/mixtral/mg/mixtral-8*7b-tp4-ep2-mcore-base"
26 26 
27TP=427TP=4
28PP=128PP=1
Mtests/st/shell_scripts/tune_llama2_tp1_pp1_qlora_ptd.sh+3-3
@@ -14,9 +14,9 @@ WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES))
14 14 
15basepath=$(cd `dirname $0`; cd ../../../; pwd)15basepath=$(cd `dirname $0`; cd ../../../; pwd)
16 16 
17-CKPT_LOAD_DIR="/data/llama2-7b-tp1-pp1-nf4"17+CKPT_LOAD_DIR="/data/ci/models/llama2/mg/llama2-7b-tp1-pp1-nf4"
18-DATA_PATH="/data/tune_dataset/alpaca"18+DATA_PATH="/data/ci/datasets/tune_dataset/alpaca"
19-TOKENIZER_MODEL="/data/hf/llama-2-7b-hf"19+TOKENIZER_MODEL="/data/ci/models/llama2/hf/llama-2-7b-hf"
20 20 
21TP=121TP=1
22PP=122PP=1
Mtests/ut/checkpoint/test_checkpoint.json+7-7
@@ -8,8 +8,8 @@
8 "target-tensor-parallel-size": "1",8 "target-tensor-parallel-size": "1",
9 "target-pipeline-parallel-size": "4",9 "target-pipeline-parallel-size": "4",
10 "target-expert-parallel-size": "8",10 "target-expert-parallel-size": "8",
11- "load-dir":"/data/hf/deepseek2_hf",11+ "load-dir":"/data/ci/models/deepseek-v2/hf/deepseek2_hf",
12- "save-dir":"/data/ci/deepseek2/mg/deepseek2_mg",12+ "save-dir":"/data/ci/cache/deepseek2_tp1pp4ep8",
13 "use-mcore-models": null,13 "use-mcore-models": null,
14 "moe-grouped-gemm": null,14 "moe-grouped-gemm": null,
15 "model-type-hf": "deepseek2",15 "model-type-hf": "deepseek2",
@@ -29,8 +29,8 @@
29 "target-tensor-parallel-size": "1",29 "target-tensor-parallel-size": "1",
30 "target-pipeline-parallel-size": "1",30 "target-pipeline-parallel-size": "1",
31 "target-expert-parallel-size": "1",31 "target-expert-parallel-size": "1",
32- "save-dir":"/data/hf/deepseek2_hf",32+ "save-dir":"/data/ci/models/deepseek-v2/hf/deepseek2_hf",
33- "load-dir":"/data/ci/deepseek2/mg_base/deepseek2-l8-t1p4e8-gemm_new",33+ "load-dir":"/data/ci/models/deepseek2/mg/deepseek2-l8-t1p4e8-gemm_new",
34 "use-mcore-models": null,34 "use-mcore-models": null,
35 "moe-grouped-gemm": null,35 "moe-grouped-gemm": null,
36 "model-type-hf": "deepseek2",36 "model-type-hf": "deepseek2",
@@ -50,9 +50,9 @@
50 "target-tensor-parallel-size": "1",50 "target-tensor-parallel-size": "1",
51 "target-pipeline-parallel-size": "2",51 "target-pipeline-parallel-size": "2",
52 "num-layers-per-virtual-pipeline-stage": "2",52 "num-layers-per-virtual-pipeline-stage": "2",
53- "load-dir":"/data/ci/orm/llama-2-7b-layers8-rm-hf/",53+ "load-dir":"/data/ci/models/llama2/hf/llama-2-7b-layers8-rm-hf/",
54- "save-dir":"/data/ci/orm/llama-2-7b-layers8-rm-mcore_pp2vpp2_test/",54+ "save-dir":"/data/ci/cache/llama-2-7b-layers8-rm-mcore_pp2vpp2_test/",
55- "tokenizer-model":"/data/hf/llama-2-7b-hftokenizer.model",55+ "tokenizer-model":"/data/ci/models/llama2/hf/llama-2-7b-hf/tokenizer.model",
56 "use-mcore-models": null,56 "use-mcore-models": null,
57 "model-type-hf": "llama2",57 "model-type-hf": "llama2",
58 "orm": null58 "orm": null
Mtests/ut/checkpoint/test_checkpoint.py+3-3
@@ -38,7 +38,7 @@ class TestCheckpoint(object):
38 os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "1"38 os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "1"
39 exit_code = run_cmd(["python3", CKPT_PYPATH] + self.test_config_cmd['test_deepseek2_hf2mcore_tp1pp4ep8'])39 exit_code = run_cmd(["python3", CKPT_PYPATH] + self.test_config_cmd['test_deepseek2_hf2mcore_tp1pp4ep8'])
40 assert exit_code == 040 assert exit_code == 0
41- base_dir = '/data/ci/deepseek2/mg_base/deepseek2-mla_tp-l8-t1p4e8-gemm_new'41+ base_dir = '/data/ci/models/deepseek2/mg/deepseek2-mla_tp-l8-t1p4e8-gemm_new'
42 save_dir = self.test_config['test_deepseek2_hf2mcore_tp1pp4ep8'][0]['save-dir']42 save_dir = self.test_config['test_deepseek2_hf2mcore_tp1pp4ep8'][0]['save-dir']
43 assert weight_compare(base_dir, save_dir)43 assert weight_compare(base_dir, save_dir)
44 shutil.rmtree(save_dir)44 shutil.rmtree(save_dir)
@@ -47,7 +47,7 @@ class TestCheckpoint(object):
47 os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "1"47 os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "1"
48 exit_code = run_cmd(["python3", CKPT_PYPATH] + self.test_config_cmd['test_deepseek2_mcore2hf_tp1pp4ep8'])48 exit_code = run_cmd(["python3", CKPT_PYPATH] + self.test_config_cmd['test_deepseek2_mcore2hf_tp1pp4ep8'])
49 assert exit_code == 049 assert exit_code == 0
50- base_dir = '/data/ci/deepseek2/hf_base/deepseek2_mla-tp_hf_base'50+ base_dir = '/data/ci/models/deepseek-v2/hf/deepseek2_mla-tp_hf_base'
51 save_dir = os.path.join(self.test_config['test_deepseek2_mcore2hf_tp1pp4ep8'][0]['save-dir'], 'mg2hf')51 save_dir = os.path.join(self.test_config['test_deepseek2_mcore2hf_tp1pp4ep8'][0]['save-dir'], 'mg2hf')
52 assert weight_compare(base_dir, save_dir, suffix="safetensors", use_md5=True)52 assert weight_compare(base_dir, save_dir, suffix="safetensors", use_md5=True)
53 shutil.rmtree(save_dir)53 shutil.rmtree(save_dir)
@@ -56,7 +56,7 @@ class TestCheckpoint(object):
56 os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "1"56 os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "1"
57 exit_code = run_cmd(["python3", CKPT_PYPATH] + self.test_config_cmd['test_llama2_hf2mcore_orm_pp2vpp2'])57 exit_code = run_cmd(["python3", CKPT_PYPATH] + self.test_config_cmd['test_llama2_hf2mcore_orm_pp2vpp2'])
58 assert exit_code == 058 assert exit_code == 0
59- base_dir = '/data/ci/orm/llama-2-7b-layers8-rm-mcore_pp2vpp2/'59+ base_dir = '/data/ci/models/llama2/mg/llama-2-7b-layers8-rm-mcore_pp2vpp2/'
60 save_dir = self.test_config['test_llama2_hf2mcore_orm_pp2vpp2'][0]['save-dir']60 save_dir = self.test_config['test_llama2_hf2mcore_orm_pp2vpp2'][0]['save-dir']
61 assert weight_compare(base_dir, save_dir)61 assert weight_compare(base_dir, save_dir)
62 shutil.rmtree(save_dir)62 shutil.rmtree(save_dir)
Mtests/ut/evaluation/test_evaluate.json+9-9
@@ -2,7 +2,7 @@
2 "test_llama2_mcore_agieval_evaluate": [2 "test_llama2_mcore_agieval_evaluate": [
3 {3 {
4 "param": {4 "param": {
5- "task-data-path":"/data/eval_data/agieval",5+ "task-data-path":"/data/ci/datasets/eval_data/agieval",
6 "task":"agieval",6 "task":"agieval",
7 "tensor-model-parallel-size": 8,7 "tensor-model-parallel-size": 8,
8 "pipeline-model-parallel-size": 1,8 "pipeline-model-parallel-size": 1,
@@ -27,9 +27,9 @@
27 "fp16": null,27 "fp16": null,
28 "no-load-rng": null,28 "no-load-rng": null,
29 "no-load-optim": null,29 "no-load-optim": null,
30- "load":"/data/ci/ckpt",30+ "load":"/data/ci/models/llama2/mg/llama2-tp8pp1",
31 "tokenizer-type":"PretrainedFromHF",31 "tokenizer-type":"PretrainedFromHF",
32- "tokenizer-name-or-path":"/data/hf/llama-2-7b-hf",32+ "tokenizer-name-or-path":"/data/ci/models/llama2/hf/llama-2-7b-hf",
33 "npu-deterministic": null33 "npu-deterministic": null
34 }34 }
35 }35 }
@@ -38,7 +38,7 @@
38 "test_llama2_mcore_bbh_evaluate": [38 "test_llama2_mcore_bbh_evaluate": [
39 {39 {
40 "param": {40 "param": {
41- "task-data-path":"/data/eval_data/bbh/test",41+ "task-data-path":"/data/ci/datasets/eval_data/bbh/test",
42 "task":"bbh",42 "task":"bbh",
43 "tensor-model-parallel-size": 8,43 "tensor-model-parallel-size": 8,
44 "pipeline-model-parallel-size": 1,44 "pipeline-model-parallel-size": 1,
@@ -63,9 +63,9 @@
63 "fp16": null,63 "fp16": null,
64 "no-load-rng": null,64 "no-load-rng": null,
65 "no-load-optim": null,65 "no-load-optim": null,
66- "load":"/data/ci/ckpt",66+ "load":"/data/ci/models/llama2/mg/llama2-tp8pp1",
67 "tokenizer-type":"PretrainedFromHF",67 "tokenizer-type":"PretrainedFromHF",
68- "tokenizer-name-or-path":"/data/hf/llama-2-7b-hf",68+ "tokenizer-name-or-path":"/data/ci/models/llama2/hf/llama-2-7b-hf",
69 "npu-deterministic": null69 "npu-deterministic": null
70 }70 }
71 }71 }
@@ -76,7 +76,7 @@
76 "param": {76 "param": {
77 "use-mcore-models": null,77 "use-mcore-models": null,
78 "use-kv-cache": null,78 "use-kv-cache": null,
79- "task-data-path": "/data/eval_data/mmlu/data/test",79+ "task-data-path": "/data/ci/datasets/eval_data/mmlu/data/test",
80 "task": "mmlu",80 "task": "mmlu",
81 "tensor-model-parallel-size": 1,81 "tensor-model-parallel-size": 1,
82 "pipeline-model-parallel-size": 1,82 "pipeline-model-parallel-size": 1,
@@ -84,7 +84,7 @@
84 "seq-length": 4096,84 "seq-length": 4096,
85 "max-position-embeddings": 4096,85 "max-position-embeddings": 4096,
86 "tokenizer-type": "PretrainedFromHF",86 "tokenizer-type": "PretrainedFromHF",
87- "tokenizer-name-or-path": "/data/Qwen2-1.5B",87+ "tokenizer-name-or-path": "/data/ci/models/qwen2-1.5b/hf/Qwen2-1.5B",
88 "max-new-tokens": 1,88 "max-new-tokens": 1,
89 "make-vocab-size-divisible-by": 1,89 "make-vocab-size-divisible-by": 1,
90 "padded-vocab-size": 151936,90 "padded-vocab-size": 151936,
@@ -99,7 +99,7 @@
99 "disable-bias-linear": null,99 "disable-bias-linear": null,
100 "swiglu": null,100 "swiglu": null,
101 "position-embedding-type": "rope",101 "position-embedding-type": "rope",
102- "load": "/data/qwen2-1.5b-hf-v0.1-tp1-pp1/",102+ "load": "/data/ci/models/qwen2-1.5b/mg/qwen2-1.5b-hf-v0.1-tp1-pp1/",
103 "normalization": "RMSNorm",103 "normalization": "RMSNorm",
104 "norm-epsilon": 1e-06,104 "norm-epsilon": 1e-06,
105 "tokenizer-not-use-fast": null,105 "tokenizer-not-use-fast": null,
Mtests/ut/inference/test_inference.json+7-7
@@ -23,10 +23,10 @@
23 "make-vocab-size-divisible-by": 1,23 "make-vocab-size-divisible-by": 1,
24 "swiglu": null,24 "swiglu": null,
25 "normalization": "RMSNorm", 25 "normalization": "RMSNorm",
26- "load":"/data/ci/ckpt",26+ "load":"/data/ci/models/llama2/mg/llama2-tp8pp1",
27 "tokenizer-type": "PretrainedFromHF", 27 "tokenizer-type": "PretrainedFromHF",
28- "tokenizer-name-or-path":"/data/hf/llama-2-7b-hf",28+ "tokenizer-name-or-path":"/data/ci/models/llama2/hf/llama-2-7b-hf",
29- "tokenizer-model": "/data/hf/llama-2-7b-hftokenizer.model",29+ "tokenizer-model": "/data/ci/models/llama2/hf/llama-2-7b-hftokenizer.model",
30 "disable-bias-linear": null,30 "disable-bias-linear": null,
31 "attention-softmax-in-fp32": null, 31 "attention-softmax-in-fp32": null,
32 "untie-embeddings-and-output-weights": null, 32 "untie-embeddings-and-output-weights": null,
@@ -61,7 +61,7 @@
61 "max-position-embeddings": 163840,61 "max-position-embeddings": 163840,
62 "position-embedding-type": "rope",62 "position-embedding-type": "rope",
63 "swiglu": null,63 "swiglu": null,
64- "load": "/data/ci/deepseek2/mg_base/deepseek2-l2-tp1-pp1-ep8-new",64+ "load": "/data/ci/models/deepseek2/mg/deepseek2-l2-tp1-pp1-ep8-new",
65 "tokenizer-type": "PretrainedFromHF",65 "tokenizer-type": "PretrainedFromHF",
66 "tokenizer-name-or-path": "/data/gemm/",66 "tokenizer-name-or-path": "/data/gemm/",
67 "bf16": null,67 "bf16": null,
@@ -149,10 +149,10 @@
149 "norm-epsilon": 1e-5,149 "norm-epsilon": 1e-5,
150 "hidden-dropout": 0,150 "hidden-dropout": 0,
151 "attention-dropout": 0,151 "attention-dropout": 0,
152- "load": "/data/llama3-8b-mcore-tp2-pp4",152+ "load": "/data/ci/models/llama3/mg/llama3-8b-mcore-tp2-pp4",
153 "tokenizer-type": "PretrainedFromHF", 153 "tokenizer-type": "PretrainedFromHF",
154- "tokenizer-name-or-path":"/data/llama-3-8b-hf",154+ "tokenizer-name-or-path":"/data/ci/models/llama3/hf/llama-3-8b-hf",
155- "tokenizer-model": "/data/llama-3-8b-hf/tokenizer.model", 155+ "tokenizer-model": "/data/ci/models/llama3/hf/llama-3-8b-hf/tokenizer.model",
156 "disable-bias-linear": null,156 "disable-bias-linear": null,
157 "attention-softmax-in-fp32": null, 157 "attention-softmax-in-fp32": null,
158 "untie-embeddings-and-output-weights": null, 158 "untie-embeddings-and-output-weights": null,
Mtests/ut/process_data/test_preprocess_data.json+26-26
@@ -2,21 +2,21 @@
2 "pretrain_dataset": [2 "pretrain_dataset": [
3 {3 {
4 "params" : {4 "params" : {
5- "input-dataset": "/data/process_dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet",5+ "input-dataset": "/data/ci/datasets/origin/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
6- "test-out-part": "/data/process_dataset/test_merge_subs/",6+ "test-out-part": "/data/ci/cache/process_dataset/test_merge_subs/",
7- "base-out-part": "/data/process_dataset/base_merge_subs/",7+ "base-out-part": "/data/ci/datasets/processed/base_merge_subs/",
8- "test-out-merge": "/data/process_dataset/test_merge/",8+ "test-out-merge": "/data/ci/cache/process_dataset/test_merge/",
9- "base-out-merge": "/data/process_dataset/base_merge/"9+ "base-out-merge": "/data/ci/datasets/processed/base_merge/"
10 }10 }
11 }11 }
12 ],12 ],
13 "test_pretrain_datasets_part1": [13 "test_pretrain_datasets_part1": [
14 {14 {
15 "params": {15 "params": {
16- "input": "/data/process_dataset/0001-alpaca.parquet",16+ "input": "/data/ci/datasets/origin/0001-alpaca.parquet",
17 "tokenizer-type": "PretrainedFromHF",17 "tokenizer-type": "PretrainedFromHF",
18- "output-prefix": "/data/process_dataset/test_merge_subs/part1", 18+ "output-prefix": "/data/ci/cache/process_dataset/test_merge_subs/part1",
19- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf",19+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf",
20 "workers": 4,20 "workers": 4,
21 "log-interval": 100021 "log-interval": 1000
22 }22 }
@@ -25,10 +25,10 @@
25 "test_pretrain_datasets_part2": [25 "test_pretrain_datasets_part2": [
26 {26 {
27 "params": {27 "params": {
28- "input": "/data/process_dataset/0002-alpaca.parquet",28+ "input": "/data/ci/datasets/origin/0002-alpaca.parquet",
29 "tokenizer-type": "PretrainedFromHF",29 "tokenizer-type": "PretrainedFromHF",
30- "output-prefix": "/data/process_dataset/test_merge_subs/part2", 30+ "output-prefix": "/data/ci/cache/process_dataset/test_merge_subs/part2",
31- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf",31+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf",
32 "workers": 4,32 "workers": 4,
33 "log-interval": 100033 "log-interval": 1000
34 }34 }
@@ -37,8 +37,8 @@
37 "test_merge_pretrain_datasets": [37 "test_merge_pretrain_datasets": [
38 {38 {
39 "params": {39 "params": {
40- "input": "/data/process_dataset/test_merge_subs/",40+ "input": "/data/ci/cache/process_dataset/test_merge_subs/",
41- "output-prefix": "/data/process_dataset/test_merge/merge",41+ "output-prefix": "/data/ci/cache/process_dataset/test_merge/merge",
42 "merge-group-keys": "text_document"42 "merge-group-keys": "text_document"
43 }43 }
44 }44 }
@@ -46,22 +46,22 @@
46 "instruction_dataset": [46 "instruction_dataset": [
47 {47 {
48 "params" : {48 "params" : {
49- "input-dataset": "/data/process_dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet",49+ "input-dataset": "/data/ci/datasets/origin/train-00000-of-00001-a09b74b3ef9c3b56.parquet",
50- "test-out-part": "/data/process_dataset/test_ins_subs/",50+ "test-out-part": "/data/ci/cache/process_dataset/test_ins_subs/",
51- "base-out-part": "/data/process_dataset/base_ins_subs/",51+ "base-out-part": "/data/ci/datasets/processed/base_ins_subs/",
52- "test-out-merge": "/data/process_dataset/test_ins_merge/",52+ "test-out-merge": "/data/ci/cache/process_dataset/test_ins_merge/",
53- "base-out-merge": "/data/process_dataset/base_ins_merge/"53+ "base-out-merge": "/data/ci/datasets/processed/base_ins_merge/"
54 }54 }
55 }55 }
56 ],56 ],
57 "test_instruction_datasets_part1": [57 "test_instruction_datasets_part1": [
58 {58 {
59 "params": {59 "params": {
60- "input": "/data/process_dataset/0001-alpaca.parquet",60+ "input": "/data/ci/datasets/origin/0001-alpaca.parquet",
61 "tokenizer-type": "PretrainedFromHF",61 "tokenizer-type": "PretrainedFromHF",
62 "handler-name": "GeneralInstructionHandler",62 "handler-name": "GeneralInstructionHandler",
63- "output-prefix": "/data/process_dataset/test_ins_subs/part1", 63+ "output-prefix": "/data/ci/cache/process_dataset/test_ins_subs/part1",
64- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf",64+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf",
65 "workers": 4,65 "workers": 4,
66 "log-interval": 1000,66 "log-interval": 1000,
67 "append-eod": null67 "append-eod": null
@@ -71,11 +71,11 @@
71 "test_instruction_datasets_part2": [71 "test_instruction_datasets_part2": [
72 {72 {
73 "params": {73 "params": {
74- "input": "/data/process_dataset/0002-alpaca.parquet",74+ "input": "/data/ci/datasets/origin/0002-alpaca.parquet",
75 "tokenizer-type": "PretrainedFromHF",75 "tokenizer-type": "PretrainedFromHF",
76 "handler-name": "GeneralInstructionHandler",76 "handler-name": "GeneralInstructionHandler",
77- "output-prefix": "/data/process_dataset/test_ins_subs/part2", 77+ "output-prefix": "/data/ci/cache/process_dataset/test_ins_subs/part2",
78- "tokenizer-name-or-path": "/data/hf/llama-2-7b-hf",78+ "tokenizer-name-or-path": "/data/ci/models/llama2/hf/llama-2-7b-hf",
79 "workers": 4,79 "workers": 4,
80 "log-interval": 1000,80 "log-interval": 1000,
81 "append-eod": null81 "append-eod": null
@@ -85,8 +85,8 @@
85 "test_merge_instrction_datasets": [85 "test_merge_instrction_datasets": [
86 {86 {
87 "params": {87 "params": {
88- "input": "/data/process_dataset/test_ins_subs/",88+ "input": "/data/ci/cache/process_dataset/test_ins_subs/",
89- "output-prefix": "/data/process_dataset/test_ins_merge/merge",89+ "output-prefix": "/data/ci/cache/process_dataset/test_ins_merge/merge",
90 "merge-group-keys": ["packed_attention_mask_document", "packed_input_ids_document", "packed_labels_document"]90 "merge-group-keys": ["packed_attention_mask_document", "packed_input_ids_document", "packed_labels_document"]
91 }91 }
92 }92 }