import torch
import torch_npu
from memcache_hybrid import DistributedObjectStore
from _pymmc import MmcCopyDirect
torch_npu.npu.set_device(6)
store = DistributedObjectStore()
store.init(0)
tensor_put = torch.ones([1024, 1024], device="npu")
# PUT
size = tensor_put.nelement() * tensor_put.element_size()
result = store.put_from("key", tensor_put.data_ptr(), size, direct=MmcCopyDirect.SMEMB_COPY_L2G)
assert result == 0print("======= put_from success")
# torch_npu.npu.set_device(1)# store = DistributedObjectStore()# store.init(1)
tensor_get = torch.zeros([1024, 1024], device="npu")
size = tensor_get.nelement() * tensor_get.element_size()
result = store.get_into("key", tensor_get.data_ptr(), size, direct=MmcCopyDirect.SMEMB_COPY_G2L)
assert result == 0print(tensor_get)
2025-12-19 11:24:15.647367 info 22127 [MMC mmc_configuration.h:347] After alignment 2MB, DRAM size is 21474836480
2025-12-19 11:24:15.647411 info 22127 [MMC mmc_configuration.h:348] After alignment 2MB, HBM size is 0
2025-12-19 11:24:15.647434 info 22127 [MMC mmc_local_service_default.cpp:25] Starting meta service local_service
2025-12-19 11:24:15.647522 info 22127 pid[22127] [SMEM smem.cpp:31] smem init successfully, library version: 1.0.1, build time: Dec 19 2025 11:16:15, commit: a1f2de1ff243a3e449638d712714bbdd53692cba
2025-12-19 11:24:15.648695 info 22329 pid[22127] [AccLink acc_tcp_link_delay_cleanup.h:116] AccDelay cleanup thread thread started
2025-12-19 11:24:15.648917 info 22330 pid[22127] [AccLink acc_tcp_worker.cpp:154] Worker [name AccWrk, index 0, cpu -1, thread-priority 0, poll-timeout-ms 500] progress thread started
2025-12-19 11:24:15.649072 info 22331 pid[22127] [AccLink acc_tcp_worker.cpp:154] Worker [name AccWrk, index 1, cpu -1, thread-priority 0, poll-timeout-ms 500] progress thread started
2025-12-19 11:24:15.649861 info 22127 pid[22127] [AccLink acc_tcp_server_default.cpp:491] Trying to connect to 61.47.1.122:6011
2025-12-19 11:24:15.650281 info 22127 pid[22127] [AccLink acc_tcp_server_default.cpp:676] Connect to 61.47.1.122:6011 successfully, with ssl disable
2025-12-19 11:24:15.650779 info 22127 pid[22127] [SMEM smem_bm_entry_manager.cpp:135] Success to auto ranking rankId: 0 localIp: deviceId: 0
2025-12-19 11:24:15.650800 info 22127 pid[22127] [SMEM smem_bm_entry_manager.cpp:72] initialize store(tcp://61.47.1.122:6011) world size(16) device(0) OK.
2025-12-19 11:24:15.650957 info 22127 pid[22127] [HYBM hybm_gva_version.cpp:192] Driver version V4 found
2025-12-19 11:24:15.652957 info 22127 pid[22127] [HYBM dl_hccp_api.cpp:126] LoadLibrary for DlHccpApi success
2025-12-19 11:24:15.653241 warn 22127 pid[22127] [HYBM dl_hcom_api.cpp:78] Failed to open library [libhcom.so], error: libhcom.so: cannot open shared object file: No such file or directory
2025-12-19 11:24:15.660562 info 22127 pid[22127] [HYBM hybm_functions.h:35] Not set rt visible envreturn deviceId: 0
2025-12-19 11:24:15.660593 info 22127 pid[22127] [HYBM hybm_gva.cpp:44] Success get deviceId: 0, logicDeviceId: 0
2025-12-19 11:24:15.905876 info 22127 pid[22127] [HYBM devmm_svm_gva.cpp:492] gva alloc heap. (size=0x40000000)
2025-12-19 11:24:15.905995 info 22127 pid[22127] [HYBM hybm_entry.cpp:107] hybm init successfully, library version: 1.0.1, build time: Dec 19 2025 11:16:19, commit: a1f2de1ff243a3e449638d712714bbdd53692cba, deviceId: 0
2025-12-19 11:24:15.906016 info 22127 pid[22127] [SMEM smem_bm.cpp:95] smem_bm_init success. config_ip: tcp://61.47.1.122:6011
2025-12-19 11:24:16.006453 info 22127 pid[22127] [HYBM hybm_functions.h:35] Not set rt visible envreturn deviceId: 0
2025-12-19 11:24:16.006482 info 22127 pid[22127] [HYBM hybm_dev_legacy_segment.cpp:407] Success get deviceId: 0, logicDeviceId: 0
2025-12-19 11:24:16.006546 info 22127 pid[22127] [HYBM hybm_functions.h:35] Not set rt visible envreturn deviceId: 0
2025-12-19 11:24:19.810301 info 22127 pid[22127] [HYBM device_rdma_transport_manager.cpp:614] initialize RDev success.
2025-12-19 11:24:19.810927 info 22127 pid[22127] [HYBM device_rdma_transport_manager.cpp:112] open device with TransportOptions(rankId=0, count=16, nic=tcp://127.0.0.1:7000) success.
2025-12-19 11:24:19.817265 info 22127 pid[22127] [HYBM hybm_data_operator.h:64] update type 2 gva: 35184372088832, space:21474836480, rankCnt:16
2025-12-19 11:24:21.868922 info 22127 pid[22127] [SMEM smem_bm_entry.cpp:140] dojoin func, local_rk: 0 receive_rk: 0, rank size is: 1
2025-12-19 11:24:21.869390 info 22127 pid[22127] [SMEM smem_net_group_engine.cpp:335] allGather successfully, key: SMEM_BM_(0)_D_1_1_GA, rank: 0, size: 1, timeCostUs: total(389) append(102) getStatus(100) getData(88)
2025-12-19 11:24:21.869817 info 22127 pid[22127] [SMEM smem_net_group_engine.cpp:335] allGather successfully, key: SMEM_BM_(0)_D_1_2_GA, rank: 0, size: 1, timeCostUs: total(385) append(97) getStatus(103) getData(90)
2025-12-19 11:24:21.870245 info 22127 pid[22127] [SMEM smem_net_group_engine.cpp:167] groupBarrier successfully, key: SMEM_BM_(0)_D_1_1_BW, size: 1, timeCostUs: total(295) add(109) getStatus(92)
2025-12-19 11:24:21.870834 info 22127 pid[22127] [SMEM smem_net_group_engine.cpp:335] allGather successfully, key: SMEM_BM_(0)_D_1_3_GA, rank: 0, size: 1, timeCostUs: total(539) append(95) getStatus(95) getData(85)
2025-12-19 11:24:21.871641 info 22127 pid[22127] [HYBM device_qp_manager.cpp:92] start to listen on port: 7000 success.
2025-12-19 11:24:21.872222 info 22127 pid[22127] [SMEM smem_net_group_engine.cpp:167] groupBarrier successfully, key: SMEM_BM_(0)_D_1_2_BW, size: 1, timeCostUs: total(317) add(125) getStatus(94)
2025-12-19 11:24:21.872449 info 22127 [MMC mmc_bm_proxy.cpp:80] init bm success, rank:0, worldSize:16, hbm{0}, dram{21474836480}
2025-12-19 11:24:21.872626 info 22127 [MMC mmc_net_ctx_store.h:68] Initialized context store, flatten capacity 65536, versionAndSeqMask 4194303, seqNoMask 65535, seqNoAndVersionIndex 1
2025-12-19 11:24:21.872844 info 22341 pid[22127] [AccLink acc_tcp_link_delay_cleanup.h:116] AccDelay cleanup thread thread started
2025-12-19 11:24:21.873080 info 22342 pid[22127] [AccLink acc_tcp_worker.cpp:154] Worker [name AccWrk, index 0, cpu -1, thread-priority 0, poll-timeout-ms 500] progress thread started
2025-12-19 11:24:21.873265 info 22343 pid[22127] [AccLink acc_tcp_worker.cpp:154] Worker [name AccWrk, index 1, cpu -1, thread-priority 0, poll-timeout-ms 500] progress thread started
2025-12-19 11:24:21.874698 info 22127 [MMC mmc_meta_net_client.cpp:67] initialize meta net server success [MetaClientCommon]
2025-12-19 11:24:21.875584 info 22127 pid[22127] [AccLink acc_tcp_server_default.cpp:491] Trying to connect to 61.47.1.122:5011
2025-12-19 11:24:21.875957 info 22127 pid[22127] [AccLink acc_tcp_server_default.cpp:676] Connect to 61.47.1.122:5011 successfully, with ssl disable
2025-12-19 11:24:21.876021 info 22127 [MMC mmc_local_service_default.cpp:146] mmc local register capacity:21474836480, type:1
2025-12-19 11:24:21.876412 info 22127 [MMC mmc_local_service_default.cpp:172] bm register succeed, bmRankId=0, type num=1
2025-12-19 11:24:21.876444 info 22127 [MMC mmc_local_service_default.cpp:74] Started LocalService (local_service) server tcp://61.47.1.122:5011
2025-12-19 11:24:21.876504 info 22127 [MMC mmc_client_default.cpp:34] Starting client mmc_client
[E compiler_depend.ts:441] operator():build/CMakeFiles/torch_npu.dir/compiler_depend.ts:46 NPU function error: call aclnnInplaceOne failed, error code is 107003
[ERROR] 2025-12-19-11:24:21 (PID:22127, Device:6, RankID:-1) ERR00100 PTA call acl api failed
[Error]: The stream is not in the current context.
Check whether the context where the stream is located is the same as the current context.
[PID: 22127] 2025-12-19-11:24:21.998.341 AclNN_Runtime_Error(EZ9903): rtKernelLaunchWithHandleV2 failed: 107003
Solution: In this scenario, collect the plog when the fault occurs and locate the fault based on the plog.
TraceBack (most recent call last):
Kernel launch with handle failed, stream is not in current ctx, stream_id=34.[FUNC:KernelLaunchWithHandle][FILE:api_impl.cc][LINE:533]
rtKernelLaunchWithHandleV2 execute failed, reason=[stream not in current context][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:53]
rtKernelLaunchWithHandleV2 failed: 107003
#### KernelLaunch failed: /usr/local/Ascend/ascend-toolkit/8.3.RC1/opp/built-in/op_impl/ai_core/tbe//kernel/ascend910b/ones_like/OnesLike_5c5e58e043224cccf51a06108010ae13_high_performance.o
Kernel Run failed. opType: 4, OnesLike
launch failed for OnesLike, errno:361001.
Exception raised from operator() at build/CMakeFiles/torch_npu.dir/compiler_depend.ts:46 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x68 (0xffff744fd898 in /usr/local/lib/python3.11/site-packages/torch/lib/libc10.so)
frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::string const&) + 0x6c (0xffff744b62a8 in /usr/local/lib/python3.11/site-packages/torch/lib/libc10.so)
frame #2: <unknown function> + 0x1805e24 (0xfffd8bad5e24 in /usr/local/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #3: <unknown function> + 0x2644e7c (0xfffd8c914e7c in /usr/local/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #4: <unknown function> + 0x8ba1d8 (0xfffd8ab8a1d8 in /usr/local/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #5: <unknown function> + 0x8bcea0 (0xfffd8ab8cea0 in /usr/local/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #6: <unknown function> + 0x8b8d3c (0xfffd8ab88d3c in /usr/local/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so)
frame #7: <unknown function> + 0x647f4c (0xffff803e7f4c in /usr/local/Ascend/ascend-toolkit/latest/lib64/libgraph_base.so)
frame #8: <unknown function> + 0x7d5b8 (0xffff808fd5b8 in /lib/aarch64-linux-gnu/libc.so.6)
frame #9: <unknown function> + 0xe5edc (0xffff80965edc in /lib/aarch64-linux-gnu/libc.so.6)
Traceback (most recent call last):
File "/home/shilinlee/workspace/scripts/test_memcache.py", line 9, in <module>
tensor_put = torch.ones([1024, 1024], device="npu")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: The Inner error is reported as above. The process exits for this inner error, and the current working operator name is aclnnInplaceOne.
Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, please set the environment variable ASCEND_LAUNCH_BLOCKING=1.
Note: ASCEND_LAUNCH_BLOCKING=1 will force ops to run in synchronous mode, resulting in performance degradation. Please unset ASCEND_LAUNCH_BLOCKING intime after debugging.
[ERROR] 2025-12-19-11:24:22 (PID:22127, Device:6, RankID:-1) ERR00100 PTA call acl api failed.
[2] 22127 segmentation fault (core dumped) python test_memcache.py
import torch import torch_npu from memcache_hybrid import DistributedObjectStore from _pymmc import MmcCopyDirect torch_npu.npu.set_device(6) store = DistributedObjectStore() store.init(0) tensor_put = torch.ones([1024, 1024], device="npu") # PUT size = tensor_put.nelement() * tensor_put.element_size() result = store.put_from("key", tensor_put.data_ptr(), size, direct=MmcCopyDirect.SMEMB_COPY_L2G) assert result == 0 print("======= put_from success") # torch_npu.npu.set_device(1) # store = DistributedObjectStore() # store.init(1) tensor_get = torch.zeros([1024, 1024], device="npu") size = tensor_get.nelement() * tensor_get.element_size() result = store.get_into("key", tensor_get.data_ptr(), size, direct=MmcCopyDirect.SMEMB_COPY_G2L) assert result == 0 print(tensor_get)2025-12-19 11:24:15.647367 info 22127 [MMC mmc_configuration.h:347] After alignment 2MB, DRAM size is 21474836480 2025-12-19 11:24:15.647411 info 22127 [MMC mmc_configuration.h:348] After alignment 2MB, HBM size is 0 2025-12-19 11:24:15.647434 info 22127 [MMC mmc_local_service_default.cpp:25] Starting meta service local_service 2025-12-19 11:24:15.647522 info 22127 pid[22127] [SMEM smem.cpp:31] smem init successfully, library version: 1.0.1, build time: Dec 19 2025 11:16:15, commit: a1f2de1ff243a3e449638d712714bbdd53692cba 2025-12-19 11:24:15.648695 info 22329 pid[22127] [AccLink acc_tcp_link_delay_cleanup.h:116] AccDelay cleanup thread thread started 2025-12-19 11:24:15.648917 info 22330 pid[22127] [AccLink acc_tcp_worker.cpp:154] Worker [name AccWrk, index 0, cpu -1, thread-priority 0, poll-timeout-ms 500] progress thread started 2025-12-19 11:24:15.649072 info 22331 pid[22127] [AccLink acc_tcp_worker.cpp:154] Worker [name AccWrk, index 1, cpu -1, thread-priority 0, poll-timeout-ms 500] progress thread started 2025-12-19 11:24:15.649861 info 22127 pid[22127] [AccLink acc_tcp_server_default.cpp:491] Trying to connect to 61.47.1.122:6011 2025-12-19 11:24:15.650281 info 22127 pid[22127] [AccLink acc_tcp_server_default.cpp:676] Connect to 61.47.1.122:6011 successfully, with ssl disable 2025-12-19 11:24:15.650779 info 22127 pid[22127] [SMEM smem_bm_entry_manager.cpp:135] Success to auto ranking rankId: 0 localIp: deviceId: 0 2025-12-19 11:24:15.650800 info 22127 pid[22127] [SMEM smem_bm_entry_manager.cpp:72] initialize store(tcp://61.47.1.122:6011) world size(16) device(0) OK. 2025-12-19 11:24:15.650957 info 22127 pid[22127] [HYBM hybm_gva_version.cpp:192] Driver version V4 found 2025-12-19 11:24:15.652957 info 22127 pid[22127] [HYBM dl_hccp_api.cpp:126] LoadLibrary for DlHccpApi success 2025-12-19 11:24:15.653241 warn 22127 pid[22127] [HYBM dl_hcom_api.cpp:78] Failed to open library [libhcom.so], error: libhcom.so: cannot open shared object file: No such file or directory 2025-12-19 11:24:15.660562 info 22127 pid[22127] [HYBM hybm_functions.h:35] Not set rt visible env return deviceId: 0 2025-12-19 11:24:15.660593 info 22127 pid[22127] [HYBM hybm_gva.cpp:44] Success get deviceId: 0, logicDeviceId: 0 2025-12-19 11:24:15.905876 info 22127 pid[22127] [HYBM devmm_svm_gva.cpp:492] gva alloc heap. (size=0x40000000) 2025-12-19 11:24:15.905995 info 22127 pid[22127] [HYBM hybm_entry.cpp:107] hybm init successfully, library version: 1.0.1, build time: Dec 19 2025 11:16:19, commit: a1f2de1ff243a3e449638d712714bbdd53692cba, deviceId: 0 2025-12-19 11:24:15.906016 info 22127 pid[22127] [SMEM smem_bm.cpp:95] smem_bm_init success. config_ip: tcp://61.47.1.122:6011 2025-12-19 11:24:16.006453 info 22127 pid[22127] [HYBM hybm_functions.h:35] Not set rt visible env return deviceId: 0 2025-12-19 11:24:16.006482 info 22127 pid[22127] [HYBM hybm_dev_legacy_segment.cpp:407] Success get deviceId: 0, logicDeviceId: 0 2025-12-19 11:24:16.006546 info 22127 pid[22127] [HYBM hybm_functions.h:35] Not set rt visible env return deviceId: 0 2025-12-19 11:24:19.810301 info 22127 pid[22127] [HYBM device_rdma_transport_manager.cpp:614] initialize RDev success. 2025-12-19 11:24:19.810927 info 22127 pid[22127] [HYBM device_rdma_transport_manager.cpp:112] open device with TransportOptions(rankId=0, count=16, nic=tcp://127.0.0.1:7000) success. 2025-12-19 11:24:19.817265 info 22127 pid[22127] [HYBM hybm_data_operator.h:64] update type 2 gva: 35184372088832, space:21474836480, rankCnt:16 2025-12-19 11:24:21.868922 info 22127 pid[22127] [SMEM smem_bm_entry.cpp:140] do join func, local_rk: 0 receive_rk: 0, rank size is: 1 2025-12-19 11:24:21.869390 info 22127 pid[22127] [SMEM smem_net_group_engine.cpp:335] allGather successfully, key: SMEM_BM_(0)_D_1_1_GA, rank: 0, size: 1, timeCostUs: total(389) append(102) getStatus(100) getData(88) 2025-12-19 11:24:21.869817 info 22127 pid[22127] [SMEM smem_net_group_engine.cpp:335] allGather successfully, key: SMEM_BM_(0)_D_1_2_GA, rank: 0, size: 1, timeCostUs: total(385) append(97) getStatus(103) getData(90) 2025-12-19 11:24:21.870245 info 22127 pid[22127] [SMEM smem_net_group_engine.cpp:167] groupBarrier successfully, key: SMEM_BM_(0)_D_1_1_BW, size: 1, timeCostUs: total(295) add(109) getStatus(92) 2025-12-19 11:24:21.870834 info 22127 pid[22127] [SMEM smem_net_group_engine.cpp:335] allGather successfully, key: SMEM_BM_(0)_D_1_3_GA, rank: 0, size: 1, timeCostUs: total(539) append(95) getStatus(95) getData(85) 2025-12-19 11:24:21.871641 info 22127 pid[22127] [HYBM device_qp_manager.cpp:92] start to listen on port: 7000 success. 2025-12-19 11:24:21.872222 info 22127 pid[22127] [SMEM smem_net_group_engine.cpp:167] groupBarrier successfully, key: SMEM_BM_(0)_D_1_2_BW, size: 1, timeCostUs: total(317) add(125) getStatus(94) 2025-12-19 11:24:21.872449 info 22127 [MMC mmc_bm_proxy.cpp:80] init bm success, rank:0, worldSize:16, hbm{0}, dram{21474836480} 2025-12-19 11:24:21.872626 info 22127 [MMC mmc_net_ctx_store.h:68] Initialized context store, flatten capacity 65536, versionAndSeqMask 4194303, seqNoMask 65535, seqNoAndVersionIndex 1 2025-12-19 11:24:21.872844 info 22341 pid[22127] [AccLink acc_tcp_link_delay_cleanup.h:116] AccDelay cleanup thread thread started 2025-12-19 11:24:21.873080 info 22342 pid[22127] [AccLink acc_tcp_worker.cpp:154] Worker [name AccWrk, index 0, cpu -1, thread-priority 0, poll-timeout-ms 500] progress thread started 2025-12-19 11:24:21.873265 info 22343 pid[22127] [AccLink acc_tcp_worker.cpp:154] Worker [name AccWrk, index 1, cpu -1, thread-priority 0, poll-timeout-ms 500] progress thread started 2025-12-19 11:24:21.874698 info 22127 [MMC mmc_meta_net_client.cpp:67] initialize meta net server success [MetaClientCommon] 2025-12-19 11:24:21.875584 info 22127 pid[22127] [AccLink acc_tcp_server_default.cpp:491] Trying to connect to 61.47.1.122:5011 2025-12-19 11:24:21.875957 info 22127 pid[22127] [AccLink acc_tcp_server_default.cpp:676] Connect to 61.47.1.122:5011 successfully, with ssl disable 2025-12-19 11:24:21.876021 info 22127 [MMC mmc_local_service_default.cpp:146] mmc local register capacity:21474836480, type:1 2025-12-19 11:24:21.876412 info 22127 [MMC mmc_local_service_default.cpp:172] bm register succeed, bmRankId=0, type num=1 2025-12-19 11:24:21.876444 info 22127 [MMC mmc_local_service_default.cpp:74] Started LocalService (local_service) server tcp://61.47.1.122:5011 2025-12-19 11:24:21.876504 info 22127 [MMC mmc_client_default.cpp:34] Starting client mmc_client [E compiler_depend.ts:441] operator():build/CMakeFiles/torch_npu.dir/compiler_depend.ts:46 NPU function error: call aclnnInplaceOne failed, error code is 107003 [ERROR] 2025-12-19-11:24:21 (PID:22127, Device:6, RankID:-1) ERR00100 PTA call acl api failed [Error]: The stream is not in the current context. Check whether the context where the stream is located is the same as the current context. [PID: 22127] 2025-12-19-11:24:21.998.341 AclNN_Runtime_Error(EZ9903): rtKernelLaunchWithHandleV2 failed: 107003 Solution: In this scenario, collect the plog when the fault occurs and locate the fault based on the plog. TraceBack (most recent call last): Kernel launch with handle failed, stream is not in current ctx, stream_id=34.[FUNC:KernelLaunchWithHandle][FILE:api_impl.cc][LINE:533] rtKernelLaunchWithHandleV2 execute failed, reason=[stream not in current context][FUNC:FuncErrorReason][FILE:error_message_manage.cc][LINE:53] rtKernelLaunchWithHandleV2 failed: 107003 #### KernelLaunch failed: /usr/local/Ascend/ascend-toolkit/8.3.RC1/opp/built-in/op_impl/ai_core/tbe//kernel/ascend910b/ones_like/OnesLike_5c5e58e043224cccf51a06108010ae13_high_performance.o Kernel Run failed. opType: 4, OnesLike launch failed for OnesLike, errno:361001. Exception raised from operator() at build/CMakeFiles/torch_npu.dir/compiler_depend.ts:46 (most recent call first): frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x68 (0xffff744fd898 in /usr/local/lib/python3.11/site-packages/torch/lib/libc10.so) frame #1: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::string const&) + 0x6c (0xffff744b62a8 in /usr/local/lib/python3.11/site-packages/torch/lib/libc10.so) frame #2: <unknown function> + 0x1805e24 (0xfffd8bad5e24 in /usr/local/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #3: <unknown function> + 0x2644e7c (0xfffd8c914e7c in /usr/local/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #4: <unknown function> + 0x8ba1d8 (0xfffd8ab8a1d8 in /usr/local/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #5: <unknown function> + 0x8bcea0 (0xfffd8ab8cea0 in /usr/local/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #6: <unknown function> + 0x8b8d3c (0xfffd8ab88d3c in /usr/local/lib/python3.11/site-packages/torch_npu/lib/libtorch_npu.so) frame #7: <unknown function> + 0x647f4c (0xffff803e7f4c in /usr/local/Ascend/ascend-toolkit/latest/lib64/libgraph_base.so) frame #8: <unknown function> + 0x7d5b8 (0xffff808fd5b8 in /lib/aarch64-linux-gnu/libc.so.6) frame #9: <unknown function> + 0xe5edc (0xffff80965edc in /lib/aarch64-linux-gnu/libc.so.6) Traceback (most recent call last): File "/home/shilinlee/workspace/scripts/test_memcache.py", line 9, in <module> tensor_put = torch.ones([1024, 1024], device="npu") ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ RuntimeError: The Inner error is reported as above. The process exits for this inner error, and the current working operator name is aclnnInplaceOne. Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, please set the environment variable ASCEND_LAUNCH_BLOCKING=1. Note: ASCEND_LAUNCH_BLOCKING=1 will force ops to run in synchronous mode, resulting in performance degradation. Please unset ASCEND_LAUNCH_BLOCKING in time after debugging. [ERROR] 2025-12-19-11:24:22 (PID:22127, Device:6, RankID:-1) ERR00100 PTA call acl api failed. [2] 22127 segmentation fault (core dumped) python test_memcache.py