{
"Tensor.__add__.0.forward": [
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 505, in _run_construct, \n output = self._run_forward_hook(inputs, output)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 98, in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)",
"File /path_to_package/site-packages/mindspore/ops/composite/multitype_ops/_compile_utils.py, line 487, in format_index_tensor, \n return F.select(index < 0, index + format_dims, index)",
"File /path_to_package/site-packages/mindspore/ops/composite/multitype_ops/_compile_utils.py, line 140, in data_update_by_ops, \n new_index = format_index_tensor(new_index, (None, F.shape(data)[:F.shape(new_index)[-1]]))",
"File /path_to_package/site-packages/mindspore/ops/composite/multitype_ops/_compile_utils.py, line 102, in data_update, \n data = data_update_by_ops(transfer_type, arg, data, new_index, origin_data, value)",
"File /path_to_package/site-packages/mindspore/ops/composite/multitype_ops/_compile_utils.py, line 203, in _tensor_getitem, \n return data_update(tensor_update_types, tensor_update_args, self, new_index)",
"File /path_to_package/site-packages/mindspore/common/tensor.py, line 483, in __getitem__, \n out = tensor_operator_registry.get('__getitem__')(self, index)",
"File /path_to_net/PanGu_ms/pangu/training/utils.py, line 65, in get_ltor_reset_masks_and_position_ids, \n eod_index = position_ids[b, data[b] == eod_token]",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 113, in get_batch, \n attention_mask, position_ids = get_ltor_reset_masks_and_position_ids(",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 724, in train, \n data = get_batch_func(train_dataset_dict_iterator)",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 329, in main, \n train(",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 342, in <module>, \n main()"
],
"Tensor.__bool__.1.forward": [
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 505, in _run_construct, \n output = self._run_forward_hook(inputs, output)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 98, in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/training/utils.py, line 75, in get_ltor_reset_masks_and_position_ids, \n if i == pre_eod_idx:",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 113, in get_batch, \n attention_mask, position_ids = get_ltor_reset_masks_and_position_ids(",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 724, in train, \n data = get_batch_func(train_dataset_dict_iterator)",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 329, in main, \n train(",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 342, in <module>, \n main()"
],
"Tensor.__add__.1.forward": [
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 505, in _run_construct, \n output = self._run_forward_hook(inputs, output)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 98, in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/training/utils.py, line 78, in get_ltor_reset_masks_and_position_ids, \n attention_mask[b, 0, (i + 1):, :(i + 1)] = 0",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 113, in get_batch, \n attention_mask, position_ids = get_ltor_reset_masks_and_position_ids(",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 724, in train, \n data = get_batch_func(train_dataset_dict_iterator)",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 329, in main, \n train(",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 342, in <module>, \n main()"
],
"Mint.logical_or.0.forward": [
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 505, in _run_construct, \n output = self._run_forward_hook(inputs, output)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 98, in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/tensor_parallel/layers.py, line 1145, in construct, \n input_mask = mint.logical_or((input_ < self.vocab_start_index), (input_ >= self.vocab_end_index))",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2455, in _backward_hook_construct, \n outputs = self.construct(outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/language_model.py, line 226, in construct, \n words_embeddings = self.word_embeddings(input_ids)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/language_model.py, line 554, in construct, \n text_embedding_out = self.embedding(enc_input_ids, enc_position_ids,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/gpt_model.py, line 101, in construct, \n lm_output = self.language_model(tokens,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/distributed/distributed_data_parallel.py, line 171, in construct, \n output = self.module(*inputs, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 747, in _complex_call, \n output = self._run_construct(*args, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 735, in __call__, \n return self._complex_call(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 113, in run_forward, \n output_tensor = model(*input_data)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 760, in forward_backward_pipelining_without_interleaving, \n micro_input_data = run_forward(*micro_input_data,",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 444, in forward_backward_with_pipelining, \n loss, logits, grads = forward_backward_pipelining_without_interleaving(",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 580, in construct, \n (loss, _), grads = self.forward_backward_func(*inputs_tuple, loss_scale=current_step_loss_scale, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 731, in __call__, \n return self.construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 725, in train, \n loss, is_finite, loss_scale, learning_rate, _ = train_one_step_cell(**data)",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 329, in main, \n train(",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 342, in <module>, \n main()"
],
"Distributed.all_reduce.0.forward": [
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 505, in _run_construct, \n output = self._run_forward_hook(inputs, output)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 98, in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/tensor_parallel/mappings.py, line 241, in construct, \n output = comm_func.all_reduce(input_, group=self.tp_group)[0]",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 785, in _call_custom_bprop, \n output = self.construct(*args, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2450, in _backward_hook_construct, \n outputs = self._call_custom_bprop(outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/tensor_parallel/layers.py, line 1168, in construct, \n output = self.reduce_from_mp_region(output_parallel)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2455, in _backward_hook_construct, \n outputs = self.construct(outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/language_model.py, line 226, in construct, \n words_embeddings = self.word_embeddings(input_ids)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/language_model.py, line 554, in construct, \n text_embedding_out = self.embedding(enc_input_ids, enc_position_ids,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/gpt_model.py, line 101, in construct, \n lm_output = self.language_model(tokens,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/distributed/distributed_data_parallel.py, line 171, in construct, \n output = self.module(*inputs, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 747, in _complex_call, \n output = self._run_construct(*args, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 735, in __call__, \n return self._complex_call(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 113, in run_forward, \n output_tensor = model(*input_data)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 760, in forward_backward_pipelining_without_interleaving, \n micro_input_data = run_forward(*micro_input_data,",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 444, in forward_backward_with_pipelining, \n loss, logits, grads = forward_backward_pipelining_without_interleaving(",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 580, in construct, \n (loss, _), grads = self.forward_backward_func(*inputs_tuple, loss_scale=current_step_loss_scale, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 731, in __call__, \n return self.construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 725, in train, \n loss, is_finite, loss_scale, learning_rate, _ = train_one_step_cell(**data)",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 329, in main, \n train(",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 342, in <module>, \n main()"
],
"Cell.network_with_loss.module.language_model.embedding.word_embeddings.VocabParallelEmbedding.forward.0": [
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 505, in _run_construct, \n output = self._run_forward_hook(inputs, output)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/language_model.py, line 226, in construct, \n words_embeddings = self.word_embeddings(input_ids)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/language_model.py, line 554, in construct, \n text_embedding_out = self.embedding(enc_input_ids, enc_position_ids,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/gpt_model.py, line 101, in construct, \n lm_output = self.language_model(tokens,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/distributed/distributed_data_parallel.py, line 171, in construct, \n output = self.module(*inputs, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 747, in _complex_call, \n output = self._run_construct(*args, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 735, in __call__, \n return self._complex_call(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 113, in run_forward, \n output_tensor = model(*input_data)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 760, in forward_backward_pipelining_without_interleaving, \n micro_input_data = run_forward(*micro_input_data,",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 444, in forward_backward_with_pipelining, \n loss, logits, grads = forward_backward_pipelining_without_interleaving(",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 580, in construct, \n (loss, _), grads = self.forward_backward_func(*inputs_tuple, loss_scale=current_step_loss_scale, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 731, in __call__, \n return self.construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 725, in train, \n loss, is_finite, loss_scale, learning_rate, _ = train_one_step_cell(**data)",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 329, in main, \n train(",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 342, in <module>, \n main()"
],
"Primitive.norm.RmsNorm.0.forward": [
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/model/transformer.py, line 198, in ParallelTransformerLayerForward, \n norm_output = self.input_norm(hidden_states)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/transformer.py, line 1454, in construct, \n hidden_states = layer(",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/language_model.py, line 579, in construct, \n encoder_output = self.encoder(encoder_input,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/gpt_model.py, line 101, in construct, \n lm_output = self.language_model(tokens,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/distributed/distributed_data_parallel.py, line 171, in construct, \n output = self.module(*inputs, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 747, in _complex_call, \n output = self._run_construct(*args, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 735, in __call__, \n return self._complex_call(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 113, in run_forward, \n output_tensor = model(*input_data)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 760, in forward_backward_pipelining_without_interleaving, \n micro_input_data = run_forward(*micro_input_data,",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 444, in forward_backward_with_pipelining, \n loss, logits, grads = forward_backward_pipelining_without_interleaving(",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 580, in construct, \n (loss, _), grads = self.forward_backward_func(*inputs_tuple, loss_scale=current_step_loss_scale, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 731, in __call__, \n return self.construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 725, in train, \n loss, is_finite, loss_scale, learning_rate, _ = train_one_step_cell(**data)",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 329, in main, \n train(",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 342, in <module>, \n main()"
],
"Cell.network_with_loss.module.language_model.encoder.layers.0.attention.ParallelAttention.forward.0": [
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 505, in _run_construct, \n output = self._run_forward_hook(inputs, output)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/model/transformer.py, line 201, in ParallelTransformerLayerForward, \n attention_output, _ = self.attention(",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/transformer.py, line 1454, in construct, \n hidden_states = layer(",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/language_model.py, line 579, in construct, \n encoder_output = self.encoder(encoder_input,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/gpt_model.py, line 101, in construct, \n lm_output = self.language_model(tokens,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/distributed/distributed_data_parallel.py, line 171, in construct, \n output = self.module(*inputs, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 747, in _complex_call, \n output = self._run_construct(*args, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 735, in __call__, \n return self._complex_call(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 113, in run_forward, \n output_tensor = model(*input_data)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 760, in forward_backward_pipelining_without_interleaving, \n micro_input_data = run_forward(*micro_input_data,",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 444, in forward_backward_with_pipelining, \n loss, logits, grads = forward_backward_pipelining_without_interleaving(",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 580, in construct, \n (loss, _), grads = self.forward_backward_func(*inputs_tuple, loss_scale=current_step_loss_scale, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 731, in __call__, \n return self.construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 725, in train, \n loss, is_finite, loss_scale, learning_rate, _ = train_one_step_cell(**data)",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 329, in main, \n train(",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 342, in <module>, \n main()"
],
"Mint.cos.0.forward": [
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 505, in _run_construct, \n output = self._run_forward_hook(inputs, output)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 98, in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/model/rotary_pos_embedding.py, line 123, in _apply_fused_rotary_pos_emb, \n cos_ = mint.cos(freqs).to(t.dtype)",
"File /path_to_net/PanGu_ms/pangu/model/rotary_pos_embedding.py, line 136, in apply_rotary_pos_emb, \n return _apply_fused_rotary_pos_emb(t, freqs)",
"File /path_to_net/PanGu_ms/pangu/model/transformer.py, line 619, in construct, \n query = apply_rotary_pos_emb(query, q_pos_emb, self.config)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/model/transformer.py, line 201, in ParallelTransformerLayerForward, \n attention_output, _ = self.attention(",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/transformer.py, line 1454, in construct, \n hidden_states = layer(",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/language_model.py, line 579, in construct, \n encoder_output = self.encoder(encoder_input,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/gpt_model.py, line 101, in construct, \n lm_output = self.language_model(tokens,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/distributed/distributed_data_parallel.py, line 171, in construct, \n output = self.module(*inputs, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 747, in _complex_call, \n output = self._run_construct(*args, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 735, in __call__, \n return self._complex_call(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 113, in run_forward, \n output_tensor = model(*input_data)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 760, in forward_backward_pipelining_without_interleaving, \n micro_input_data = run_forward(*micro_input_data,",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 444, in forward_backward_with_pipelining, \n loss, logits, grads = forward_backward_pipelining_without_interleaving(",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 580, in construct, \n (loss, _), grads = self.forward_backward_func(*inputs_tuple, loss_scale=current_step_loss_scale, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 731, in __call__, \n return self.construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 725, in train, \n loss, is_finite, loss_scale, learning_rate, _ = train_one_step_cell(**data)",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 329, in main, \n train(",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 342, in <module>, \n main()"
],
"Functional.flash_attention_score.0.forward": [
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 505, in _run_construct, \n output = self._run_forward_hook(inputs, output)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py, line 48, in __call__, \n out = super(HOOKCell, self).__call__(*args, **kwargs)",
"File /path_to_package/mstt/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_api.py, line 98, in api_function, \n return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/model/transformer.py, line 637, in construct, \n output = ops.flash_attention_score(",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/model/transformer.py, line 201, in ParallelTransformerLayerForward, \n attention_output, _ = self.attention(",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/transformer.py, line 1454, in construct, \n hidden_states = layer(",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/transformer/language_model.py, line 579, in construct, \n encoder_output = self.encoder(encoder_input,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/gpt_model.py, line 101, in construct, \n lm_output = self.language_model(tokens,",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 733, in __call__, \n return self._run_construct(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/distributed/distributed_data_parallel.py, line 171, in construct, \n output = self.module(*inputs, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 2453, in _backward_hook_construct, \n outputs = self.construct(*outputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 494, in _run_construct, \n output = self._backward_hook_construct(*inputs, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 747, in _complex_call, \n output = self._run_construct(*args, **kwargs)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 735, in __call__, \n return self._complex_call(*args, **kwargs)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 113, in run_forward, \n output_tensor = model(*input_data)",
"File /path_to_net/third_party/dynamic-parallel/mindformers/experimental/parallel_core/pynative/pipeline_parallel/schedules.py, line 760, in forward_backward_pipelining_without_interleaving, \n micro_input_data = run_forward(*micro_input_data,",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 444, in forward_backward_with_pipelining, \n loss, logits, grads = forward_backward_pipelining_without_interleaving(",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 580, in construct, \n (loss, _), grads = self.forward_backward_func(*inputs_tuple, loss_scale=current_step_loss_scale, **inputs_dict)",
"File /path_to_package/site-packages/mindspore/nn/cell.py, line 731, in __call__, \n return self.construct(*args, **kwargs)",
"File /path_to_net/PanGu_ms/pangu/pynative/training/training.py, line 725, in train, \n loss, is_finite, loss_scale, learning_rate, _ = train_one_step_cell(**data)",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 329, in main, \n train(",
"File /path_to_net/PanGu_ms/pretrain_gpt.py, line 342, in <module>, \n main()"
]
}
