3-pytorch-AtomGit

已开启

3 #3

rmch创建于 7月23日

rmch成员

7月23日创建

File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/managers/scheduler.py", line 2454, in run_scheduler_process
scheduler = Scheduler(
^^^^^^^^^^
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/managers/scheduler.py", line 350, in init
self.draft_worker = WorkerClass(
^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/speculative/eagle_worker_overlap.py", line 178, in init
self.worker.target_worker.model_runner.init_npu_graphs()
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/model_executor/model_runner.py", line 398, in init_npu_graphs
self.device_graph_runner = NpuGraphRunner(self)
^^^^^^^^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/model_executor/npu_graph_runner.py", line 62, in init
super().init(model_runner)
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/model_executor/cuda_graph_runner.py", line 308, in init
self.warm_up()
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/model_executor/npu_graph_runner.py", line 67, in warm_up
self.run_fake()
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/model_executor/npu_graph_runner.py", line 206, in run_fake
run_once()
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/model_executor/npu_graph_runner.py", line 194, in run_once
compile_forward(
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 845, in compile_wrapper
raise e.remove_dynamo_frames() from None # see TORCHDYNAMO_VERBOSE=1
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/_dynamo/output_graph.py", line 2196, in _call_user_compiler
raise BackendCompilerFailed(
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/_dynamo/output_graph.py", line 2171, in call_user_compiler
compiled_fn = compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/dynamo/repro/after_dynamo.py", line 156, in call
compiled_gm = compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/init.py", line 2437, in call
return self.compiler_fn(model, inputs, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/_dynamo/backends/debugging.py", line 215, in aot_eager
return aot_autograd(
^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/_dynamo/backends/common.py", line 117, in call
cg = aot_module_simplified(gm, example_inputs, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 1096, in aot_module_simplified
aot_state = create_aot_state(
^^^^^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 567, in create_aot_state
fw_metadata = run_functionalized_fw_and_collect_metadata(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/collect_metadata_analysis.py", line 207, in inner
flat_f_outs = f(*flat_f_args)
^^^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/graph_capture_wrappers.py", line 1352, in functional_call
out = PropagateUnbackedSymInts(mod).run(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/fx/interpreter.py", line 201, in run
raise RuntimeError(*e.args) from e
torch._dynamo.exc.BackendCompilerFailed: backend='aot_eager' raised:
RuntimeError: <weakref at 0x7f7584f06930; to 'torch.storage.UntypedStorage' at 0x7f75844d2710>

While executing %output : [num_users=1] = call_function[target=torch._C.nn.linear](args = (%hidden_states_1, %l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_0_modules_indexer_modules_wk_parameters_weight, None), kwargs = {})
Original traceback:
File "", line 3, in forward_bs_2
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/models/npu_longcat_flash.py", line 1201, in forward
hidden_states = self.model(input_ids, positions, forward_batch)
File "/workdir/npu_dev_test/test/venvtorch29/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
return func(*args, **kwargs)
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/models/npu_longcat_flash.py", line 967, in forward
hidden_states = layer(
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/models/npu_longcat_flash.py", line 248, in forward
return self.forward_decode(rotary_emb, positions, hidden_states, kv_cache, forward_batch)
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/models/npu_longcat_flash.py", line 381, in forward_decode
hidden_states, attn_out_before_comm = self.self_attn[i](
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/layers/attention/npu_attn/deepseek_mla.py", line 953, in forward
topk_indices, s = self.forward_indexer_mla_prolog(
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/layers/attention/npu_attn/deepseek_mla.py", line 834, in forward_indexer_mla_prolog
k_proj = self.indexer.wk(hidden_states)[0]
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/layers/linear.py", line 204, in forward
output = self.quant_method.apply(self, x, bias)
File "/workdir/npu_dev_test/test/pro_run_commitid_atoeager/fluentllm/python/sglang/srt/layers/dense/layouts/unquant.py", line 62, in apply
return F.linear(x, weight, bias)

Use tlparse to see full graph. (https://github.com/pytorch/tlparse?tab=readme-ov-file#tlparse-parse-structured-pt2-logs)

rmch成员

7月23日评论：

sub_1 = l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_padded_org_vocab_end_index - l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_padded_org_vocab_start_index
sub_2 = l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_padded_org_vocab_end_index - l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_padded_org_vocab_start_index
sub_3 = sub_1 - sub_2;  sub_1 = sub_2 = None
ge_2 = output_tensor_8 >= l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_padded_org_vocab_start_index
lt_2 = output_tensor_8 < l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_padded_org_vocab_end_index
org_vocab_mask_1 = ge_2 & lt_2;  ge_2 = lt_2 = None
ge_3 = output_tensor_8 >= l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_added_vocab_start_index
lt_3 = output_tensor_8 < l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_added_vocab_end_index;  l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_added_vocab_end_index = None
added_vocab_mask_1 = ge_3 & lt_3;  ge_3 = lt_3 = None
sub_4 = l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_padded_org_vocab_end_index - l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_padded_org_vocab_start_index;  l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_padded_org_vocab_end_index = None
sub_5 = l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_added_vocab_start_index - sub_4;  l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_added_vocab_start_index = sub_4 = None
added_offset = sub_5 - sub_3;  sub_5 = sub_3 = None
mul_3 = l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_padded_org_vocab_start_index * org_vocab_mask_1;  l_self_modules_model_modules_over_embedding_modules_oe_embeder_shard_indices_padded_org_vocab_start_index = None
mul_4 = added_offset * added_vocab_mask_1;  added_offset = None
valid_offset_1 = mul_3 + mul_4;  mul_3 = mul_4 = None
vocab_mask_1 = org_vocab_mask_1 | added_vocab_mask_1;  org_vocab_mask_1 = added_vocab_mask_1 = None
sub_7 = output_tensor_8 - valid_offset_1;  output_tensor_8 = valid_offset_1 = None
input__1 = vocab_mask_1 * sub_7;  sub_7 = None
input_mask_1 = ~vocab_mask_1;  vocab_mask_1 = None
long_1 = input__1.long();  input__1 = None
output_parallel_1 = torch.nn.functional.embedding(long_1, l_self_modules_model_modules_over_embedding_modules_oe_embeder_parameters_weight_);  long_1 = l_self_modules_model_modules_over_embedding_modules_oe_embeder_parameters_weight_ = None
unsqueeze_1 = input_mask_1.unsqueeze(-1);  input_mask_1 = None
masked_fill__1 = output_parallel_1.masked_fill_(unsqueeze_1, 0);  unsqueeze_1 = masked_fill__1 = None
output_tensor_9 = torch.empty((128, 512), dtype = torch.bfloat16, device = device(type='npu', index=6))
tensor_3 = torch.ops._c10d_functional.reduce_scatter_tensor(output_parallel_1, 'sum', 128, '3');  output_parallel_1 = None
res_3 = torch.ops._c10d_functional.wait_tensor(tensor_3);  tensor_3 = None
copy__3 = output_tensor_9.copy_(res_3);  res_3 = copy__3 = None
oe_hidden_states = output_tensor_9.view(16, -1, 512);  output_tensor_9 = None
truediv = output_tensor_4 / 17;  output_tensor_4 = None
bmm = torch.bmm(oe_hidden_states, l_self_modules_model_modules_over_embedding_parameters_oe_projection_);  oe_hidden_states = l_self_modules_model_modules_over_embedding_parameters_oe_projection_ = None
sum_1 = bmm.sum(dim = 0);  bmm = None
hidden_states = truediv + sum_1;  truediv = sum_1 = None
cos = l_self_modules_model_modules_rotary_pos_emb_buffers_cos_cached_.to(torch.bfloat16)
sin = l_self_modules_model_modules_rotary_pos_emb_buffers_sin_cached_.to(torch.bfloat16)
cos_1 = cos[l_positions_];  cos = None
sin_1 = sin[l_positions_];  sin = None
cos_2 = cos_1.unsqueeze(1);  cos_1 = None
sin_2 = sin_1.unsqueeze(1);  sin_1 = None
cos_3 = l_self_modules_model_modules_rotary_pos_emb_buffers_cos_cached_[l_positions_];  l_self_modules_model_modules_rotary_pos_emb_buffers_cos_cached_ = None
sin_3 = l_self_modules_model_modules_rotary_pos_emb_buffers_sin_cached_[l_positions_];  l_self_modules_model_modules_rotary_pos_emb_buffers_sin_cached_ = l_positions_ = None
cos_4 = cos_3.view(-1, 1, 1, 64);  cos_3 = None
sin_4 = sin_3.view(-1, 1, 1, 64);  sin_3 = None
scope_enter = torch.ops.air.scope_enter(['_op_aicore_num', '_op_vectorcore_num'], ['12', '24']);  scope_enter = None
npu_rms_norm = torch.ops.npu.npu_rms_norm(hidden_states, l_self_modules_model_modules_layers_modules_0_modules_input_layernorm_modules_0_parameters_weight_, 1e-05);  l_self_modules_model_modules_layers_modules_0_modules_input_layernorm_modules_0_parameters_weight_ = None
hidden_states_1 = npu_rms_norm[0];  npu_rms_norm = None
scope_enter_1 = torch.ops.air.scope_enter(['_op_aicore_num', '_op_vectorcore_num'], ['12', '24']);  scope_enter_1 = None
output = torch._C._nn.linear(hidden_states_1, l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_0_modules_indexer_modules_wk_parameters_weight_, None);  l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_0_modules_indexer_modules_wk_parameters_weight_ = None
x_fp32 = output.to(torch.float32);  output = None
w_fp32 = l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_0_modules_indexer_modules_k_norm_parameters_weight_.to(torch.float32);  l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_0_modules_indexer_modules_k_norm_parameters_weight_ = None
npu_rms_norm_1 = torch.ops.npu.npu_rms_norm(x_fp32, w_fp32, 1e-06);  x_fp32 = w_fp32 = None
getitem_6 = npu_rms_norm_1[0];  npu_rms_norm_1 = None
k = getitem_6.to(torch.bfloat16);  getitem_6 = None
split = torch.functional.split(k, [64, 64], dim = -1);  k = None
k_pe = split[0]