已合并
修复DCCI及Sync类接口修改 #1756
pangfd创建于 4月23日
修复DCCI及Sync类接口修改 #1756
已合并
共 15 个文件变更+326-189
| @@ -25,21 +25,26 @@ Scalar单元访问Global Memory,首先会访问每个核内的Data Cache,因 | |||
| 25 | 25 | ||
| 26 | Scalar单元访问UB数据时,该接口需配合[asc_set_ctrl()](../sys_var/asc_set_ctrl.md)接口使用,将CTRL[49]设置为1'b1,开启datacache模式。 | 26 | Scalar单元访问UB数据时,该接口需配合[asc_set_ctrl()](../sys_var/asc_set_ctrl.md)接口使用,将CTRL[49]设置为1'b1,开启datacache模式。 |
| 27 | 27 | ||
| 28 | +<cann-filter npu_type = "950"> | ||
| 29 | + | ||
| 30 | +针对Ascend 950PR/Ascend 950DT,不支持asc_dcci_entire_ub接口。 | ||
| 31 | + | ||
| 32 | +</cann-filter> | ||
| 33 | + | ||
| 28 | ## 函数原型 | 34 | ## 函数原型 |
| 29 | 35 | ||
| 30 | ```cpp | 36 | ```cpp |
| 31 | // 刷新传入地址所在的单个Cache Line | 37 | // 刷新传入地址所在的单个Cache Line |
| 32 | __aicore__ inline void asc_dcci_single(__gm__ void* dst) | 38 | __aicore__ inline void asc_dcci_single(__gm__ void* dst) |
| 33 | - __aicore__ inline void asc_dcci_single_all(__gm__ void* dst) | 39 | + |
| 34 | - __aicore__ inline void asc_dcci_single_ub(__gm__ void* dst) | 40 | + // 刷新传入的UB地址空间地址所在的单个Cache Line |
| 35 | - __aicore__ inline void asc_dcci_single_out(__gm__ void* dst) | 41 | + __aicore__ inline void asc_ub_dcci_single(__ubuf__ void* dst) |
| 36 | - __aicore__ inline void asc_dcci_single_atomic(__gm__ void* dst) | 42 | + |
| 37 | // 刷新所有Cache Line | 43 | // 刷新所有Cache Line |
| 38 | - __aicore__ inline void asc_dcci_entire(__gm__ void* dst) | 44 | + __aicore__ inline void asc_dcci_entire_all() |
| 39 | - __aicore__ inline void asc_dcci_entire_all(__gm__ void* dst) | 45 | + __aicore__ inline void asc_dcci_entire_ub() |
| 40 | - __aicore__ inline void asc_dcci_entire_ub(__gm__ void* dst) | 46 | + __aicore__ inline void asc_dcci_entire_out() |
| 41 | - __aicore__ inline void asc_dcci_entire_out(__gm__ void* dst) | 47 | + __aicore__ inline void asc_dcci_entire_atomic() |
| 42 | - __aicore__ inline void asc_dcci_entire_atomic(__gm__ void* dst) | ||
| 43 | ``` | 48 | ``` |
| 44 | 49 | ||
| 45 | ## 参数说明 | 50 | ## 参数说明 |
| @@ -56,14 +61,9 @@ Scalar单元访问UB数据时,该接口需配合[asc_set_ctrl()](../sys_var/as | |||
| 56 | 61 | ||
| 57 | PIPE_S | 62 | PIPE_S |
| 58 | 63 | ||
| 59 | - | ||
| 60 | -<cann-filter npu_type = "950"> | ||
| 61 | - | ||
| 62 | ## 约束说明 | 64 | ## 约束说明 |
| 63 | 65 | ||
| 64 | -针对Ascend 950PR/Ascend 950DT,不支持asc_dcci_single_ub、asc_dcci_entire_ub接口。 | 66 | +无 |
| 65 | - | ||
| 66 | -</cann-filter> | ||
| 67 | 67 | ||
| 68 | ## 调用示例 | 68 | ## 调用示例 |
| 69 | 69 | ||
| @@ -4,6 +4,7 @@ | |||
| 4 | 4 | ||
| 5 | | 产品 | 是否支持 | | 5 | | 产品 | 是否支持 | |
| 6 | | :-----------| :------: | | 6 | | :-----------| :------: | |
| 7 | +| <cann-filter npu_type="950">Ascend 950PR/Ascend 950DT | √ </cann-filter>| | ||
| 7 | | Atlas A3 训练系列产品/Atlas A3 推理系列产品 | √ | | 8 | | Atlas A3 训练系列产品/Atlas A3 推理系列产品 | √ | |
| 8 | | Atlas A2 训练系列产品/Atlas A2 推理系列产品 | √ | | 9 | | Atlas A2 训练系列产品/Atlas A2 推理系列产品 | √ | |
| 9 | 10 | ||
| @@ -4,6 +4,7 @@ | |||
| 4 | 4 | ||
| 5 | | 产品 | 是否支持 | | 5 | | 产品 | 是否支持 | |
| 6 | | :-----------| :------: | | 6 | | :-----------| :------: | |
| 7 | +| <cann-filter npu_type="950">Ascend 950PR/Ascend 950DT | √ </cann-filter>| | ||
| 7 | | Atlas A3 训练系列产品/Atlas A3 推理系列产品 | √ | | 8 | | Atlas A3 训练系列产品/Atlas A3 推理系列产品 | √ | |
| 8 | | Atlas A2 训练系列产品/Atlas A2 推理系列产品 | √ | | 9 | | Atlas A2 训练系列产品/Atlas A2 推理系列产品 | √ | |
| 9 | 10 | ||
| @@ -4,6 +4,7 @@ | |||
| 4 | 4 | ||
| 5 | | 产品 | 是否支持 | | 5 | | 产品 | 是否支持 | |
| 6 | | :-----------| :------: | | 6 | | :-----------| :------: | |
| 7 | +| <cann-filter npu_type="950">Ascend 950PR/Ascend 950DT | √ </cann-filter>| | ||
| 7 | | Atlas A3 训练系列产品/Atlas A3 推理系列产品 | √ | | 8 | | Atlas A3 训练系列产品/Atlas A3 推理系列产品 | √ | |
| 8 | | Atlas A2 训练系列产品/Atlas A2 推理系列产品 | √ | | 9 | | Atlas A2 训练系列产品/Atlas A2 推理系列产品 | √ | |
| 9 | 10 | ||
| @@ -4,6 +4,7 @@ | |||
| 4 | 4 | ||
| 5 | | 产品 | 是否支持 | | 5 | | 产品 | 是否支持 | |
| 6 | | :-----------| :------: | | 6 | | :-----------| :------: | |
| 7 | +| <cann-filter npu_type="950">Ascend 950PR/Ascend 950DT | √ </cann-filter>| | ||
| 7 | | Atlas A3 训练系列产品/Atlas A3 推理系列产品 | √ | | 8 | | Atlas A3 训练系列产品/Atlas A3 推理系列产品 | √ | |
| 8 | | Atlas A2 训练系列产品/Atlas A2 推理系列产品 | √ | | 9 | | Atlas A2 训练系列产品/Atlas A2 推理系列产品 | √ | |
| 9 | 10 | ||
| @@ -28,49 +28,29 @@ __aicore__ inline void asc_dcci_single(__gm__ void* dst) | |||
| 28 | asc_dcci_single_impl(dst); | 28 | asc_dcci_single_impl(dst); |
| 29 | } | 29 | } |
| 30 | 30 | ||
| 31 | -__aicore__ inline void asc_dcci_entire(__gm__ void* dst) | 31 | +__aicore__ inline void asc_ub_dcci_single(__ubuf__ void* dst) |
| 32 | { | 32 | { |
| 33 | - asc_dcci_entire_impl(dst); | 33 | + asc_ub_dcci_single_impl(dst); |
| 34 | } | 34 | } |
| 35 | 35 | ||
| 36 | -__aicore__ inline void asc_dcci_single_all(__gm__ void* dst) | 36 | +__aicore__ inline void asc_dcci_entire_all() |
| 37 | { | 37 | { |
| 38 | - asc_dcci_single_all_impl(dst); | 38 | + asc_dcci_entire_all_impl(); |
| 39 | } | 39 | } |
| 40 | 40 | ||
| 41 | -__aicore__ inline void asc_dcci_single_ub(__gm__ void* dst) | 41 | +__aicore__ inline void asc_dcci_entire_ub() |
| 42 | { | 42 | { |
| 43 | - asc_dcci_single_ub_impl(dst); | 43 | + asc_dcci_entire_ub_impl(); |
| 44 | } | 44 | } |
| 45 | 45 | ||
| 46 | -__aicore__ inline void asc_dcci_single_out(__gm__ void* dst) | 46 | +__aicore__ inline void asc_dcci_entire_out() |
| 47 | { | 47 | { |
| 48 | - asc_dcci_single_out_impl(dst); | 48 | + asc_dcci_entire_out_impl(); |
| 49 | } | 49 | } |
| 50 | 50 | ||
| 51 | -__aicore__ inline void asc_dcci_single_atomic(__gm__ void* dst) | 51 | +__aicore__ inline void asc_dcci_entire_atomic() |
| 52 | { | 52 | { |
| 53 | - asc_dcci_single_atomic_impl(dst); | 53 | + asc_dcci_entire_atomic_impl(); |
| 54 | -} | ||
| 55 | - | ||
| 56 | -__aicore__ inline void asc_dcci_entire_all(__gm__ void* dst) | ||
| 57 | -{ | ||
| 58 | - asc_dcci_entire_all_impl(dst); | ||
| 59 | -} | ||
| 60 | - | ||
| 61 | -__aicore__ inline void asc_dcci_entire_ub(__gm__ void* dst) | ||
| 62 | -{ | ||
| 63 | - asc_dcci_entire_ub_impl(dst); | ||
| 64 | -} | ||
| 65 | - | ||
| 66 | -__aicore__ inline void asc_dcci_entire_out(__gm__ void* dst) | ||
| 67 | -{ | ||
| 68 | - asc_dcci_entire_out_impl(dst); | ||
| 69 | -} | ||
| 70 | - | ||
| 71 | -__aicore__ inline void asc_dcci_entire_atomic(__gm__ void* dst) | ||
| 72 | -{ | ||
| 73 | - asc_dcci_entire_atomic_impl(dst); | ||
| 74 | } | 54 | } |
| 75 | 55 | ||
| 76 | __aicore__ inline void asc_datacache_preload(__gm__ uint64_t* address, int64_t offset) | 56 | __aicore__ inline void asc_datacache_preload(__gm__ uint64_t* address, int64_t offset) |
| @@ -94,4 +74,3 @@ __aicore__ inline void asc_icache_preload(const void* addr, int64_t prefetch_len | |||
| 94 | 74 | ||
| 95 | 75 | ||
| 96 | 76 | ||
| 97 | - | ||
| @@ -30,55 +30,34 @@ __aicore__ inline void asc_dcci_single_impl(__gm__ void* dst) | |||
| 30 | dcci(dst, cache_line_t::SINGLE_CACHE_LINE); | 30 | dcci(dst, cache_line_t::SINGLE_CACHE_LINE); |
| 31 | } | 31 | } |
| 32 | 32 | ||
| 33 | -__aicore__ inline void asc_dcci_entire_impl(__gm__ void* dst) | 33 | +__aicore__ inline void asc_ub_dcci_single_impl(__ubuf__ void* dst) |
| 34 | { | 34 | { |
| 35 | - dcci(dst, cache_line_t::ENTIRE_DATA_CACHE); | 35 | + dcci(dst, cache_line_t::SINGLE_CACHE_LINE); |
| 36 | } | 36 | } |
| 37 | 37 | ||
| 38 | -__aicore__ inline void asc_dcci_single_all_impl(__gm__ void* dst) | 38 | +__aicore__ inline void asc_dcci_entire_all_impl() |
| 39 | { | 39 | { |
| 40 | - dcci(dst, cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_ALL); | 40 | + dcci((__gm__ int64_t*)0, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_ALL); |
| 41 | } | 41 | } |
| 42 | 42 | ||
| 43 | -__aicore__ inline void asc_dcci_single_ub_impl(__gm__ void* dst) | 43 | +__aicore__ inline void asc_dcci_entire_ub_impl() |
| 44 | { | 44 | { |
| 45 | - dcci(dst, cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_UB); | 45 | + dcci((__ubuf__ int64_t*)0, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_UB); |
| 46 | } | 46 | } |
| 47 | 47 | ||
| 48 | -__aicore__ inline void asc_dcci_single_out_impl(__gm__ void* dst) | 48 | +__aicore__ inline void asc_dcci_entire_out_impl() |
| 49 | { | 49 | { |
| 50 | - dcci(dst, cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_OUT); | 50 | + dcci((__gm__ int64_t*)0, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_OUT); |
| 51 | } | 51 | } |
| 52 | 52 | ||
| 53 | -__aicore__ inline void asc_dcci_single_atomic_impl(__gm__ void* dst) | 53 | +__aicore__ inline void asc_dcci_entire_atomic_impl() |
| 54 | { | 54 | { |
| 55 | - dcci(dst, cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_ATOMIC); | 55 | + dcci((__gm__ int64_t*)0, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_ATOMIC); |
| 56 | } | 56 | } |
| 57 | - | 57 | + |
| 58 | -__aicore__ inline void asc_dcci_entire_all_impl(__gm__ void* dst) | ||
| 59 | -{ | ||
| 60 | - dcci(dst, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_ALL); | ||
| 61 | -} | ||
| 62 | - | ||
| 63 | -__aicore__ inline void asc_dcci_entire_ub_impl(__gm__ void* dst) | ||
| 64 | -{ | ||
| 65 | - dcci(dst, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_UB); | ||
| 66 | -} | ||
| 67 | - | ||
| 68 | -__aicore__ inline void asc_dcci_entire_out_impl(__gm__ void* dst) | ||
| 69 | -{ | ||
| 70 | - dcci(dst, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_OUT); | ||
| 71 | -} | ||
| 72 | - | ||
| 73 | -__aicore__ inline void asc_dcci_entire_atomic_impl(__gm__ void* dst) | ||
| 74 | -{ | ||
| 75 | - dcci(dst, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_ATOMIC); | ||
| 76 | -} | ||
| 77 | - | ||
| 78 | 58 | ||
| 79 | 59 | ||
| 80 | 60 | ||
| 81 | 61 | ||
| 82 | 62 | ||
| 83 | 63 | ||
| 84 | - | ||
| @@ -27,45 +27,29 @@ __aicore__ inline void asc_datacache_preload(__gm__ uint64_t* address, int64_t o | |||
| 27 | asc_datacache_preload_impl(address, offset); | 27 | asc_datacache_preload_impl(address, offset); |
| 28 | } | 28 | } |
| 29 | 29 | ||
| 30 | -//==============asc_dcci=============== | ||
| 31 | __aicore__ inline void asc_dcci_single(__gm__ void* dst) | 30 | __aicore__ inline void asc_dcci_single(__gm__ void* dst) |
| 32 | { | 31 | { |
| 33 | asc_dcci_single_impl(dst); | 32 | asc_dcci_single_impl(dst); |
| 34 | } | 33 | } |
| 35 | 34 | ||
| 36 | -__aicore__ inline void asc_dcci_entire(__gm__ void* dst) | 35 | +__aicore__ inline void asc_ub_dcci_single(__ubuf__ void* dst) |
| 37 | { | 36 | { |
| 38 | - asc_dcci_entire_impl(dst); | 37 | + asc_ub_dcci_single_impl(dst); |
| 39 | } | 38 | } |
| 40 | 39 | ||
| 41 | -__aicore__ inline void asc_dcci_single_all(__gm__ void* dst) | 40 | +__aicore__ inline void asc_dcci_entire_all() |
| 42 | { | 41 | { |
| 43 | - asc_dcci_single_all_impl(dst); | 42 | + asc_dcci_entire_all_impl(); |
| 44 | } | 43 | } |
| 45 | 44 | ||
| 46 | -__aicore__ inline void asc_dcci_single_out(__gm__ void* dst) | 45 | +__aicore__ inline void asc_dcci_entire_out() |
| 47 | { | 46 | { |
| 48 | - asc_dcci_single_out_impl(dst); | 47 | + asc_dcci_entire_out_impl(); |
| 49 | } | 48 | } |
| 50 | 49 | ||
| 51 | -__aicore__ inline void asc_dcci_single_atomic(__gm__ void* dst) | 50 | +__aicore__ inline void asc_dcci_entire_atomic() |
| 52 | { | 51 | { |
| 53 | - asc_dcci_single_atomic_impl(dst); | 52 | + asc_dcci_entire_atomic_impl(); |
| 54 | -} | ||
| 55 | - | ||
| 56 | -__aicore__ inline void asc_dcci_entire_all(__gm__ void* dst) | ||
| 57 | -{ | ||
| 58 | - asc_dcci_entire_all_impl(dst); | ||
| 59 | -} | ||
| 60 | - | ||
| 61 | -__aicore__ inline void asc_dcci_entire_out(__gm__ void* dst) | ||
| 62 | -{ | ||
| 63 | - asc_dcci_entire_out_impl(dst); | ||
| 64 | -} | ||
| 65 | - | ||
| 66 | -__aicore__ inline void asc_dcci_entire_atomic(__gm__ void* dst) | ||
| 67 | -{ | ||
| 68 | - asc_dcci_entire_atomic_impl(dst); | ||
| 69 | } | 53 | } |
| 70 | 54 | ||
| 71 | //=========asc_get_icache_preload_status======== | 55 | //=========asc_get_icache_preload_status======== |
| @@ -30,45 +30,29 @@ __aicore__ inline void asc_dcci_single_impl(__gm__ void* dst) | |||
| 30 | dcci(dst, cache_line_t::SINGLE_CACHE_LINE); | 30 | dcci(dst, cache_line_t::SINGLE_CACHE_LINE); |
| 31 | } | 31 | } |
| 32 | 32 | ||
| 33 | -__aicore__ inline void asc_dcci_entire_impl(__gm__ void* dst) | 33 | +__aicore__ inline void asc_ub_dcci_single_impl(__ubuf__ void* dst) |
| 34 | { | 34 | { |
| 35 | - dcci(dst, cache_line_t::ENTIRE_DATA_CACHE); | 35 | + dcci(dst, cache_line_t::SINGLE_CACHE_LINE); |
| 36 | } | 36 | } |
| 37 | 37 | ||
| 38 | -__aicore__ inline void asc_dcci_single_all_impl(__gm__ void* dst) | 38 | +__aicore__ inline void asc_dcci_entire_all_impl() |
| 39 | { | 39 | { |
| 40 | - dcci(dst, cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_ALL); | 40 | + dcci((__gm__ int64_t*)0, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_ALL); |
| 41 | } | 41 | } |
| 42 | 42 | ||
| 43 | -__aicore__ inline void asc_dcci_single_out_impl(__gm__ void* dst) | 43 | +__aicore__ inline void asc_dcci_entire_out_impl() |
| 44 | { | 44 | { |
| 45 | - dcci(dst, cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_OUT); | 45 | + dcci((__gm__ int64_t*)0, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_OUT); |
| 46 | } | 46 | } |
| 47 | 47 | ||
| 48 | -__aicore__ inline void asc_dcci_single_atomic_impl(__gm__ void* dst) | 48 | +__aicore__ inline void asc_dcci_entire_atomic_impl() |
| 49 | { | 49 | { |
| 50 | - dcci(dst, cache_line_t::SINGLE_CACHE_LINE, dcci_dst_t::CACHELINE_ATOMIC); | 50 | + dcci((__gm__ int64_t*)0, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_ATOMIC); |
| 51 | } | 51 | } |
| 52 | 52 | ||
| 53 | -__aicore__ inline void asc_dcci_entire_all_impl(__gm__ void* dst) | ||
| 54 | -{ | ||
| 55 | - dcci(dst, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_ALL); | ||
| 56 | -} | ||
| 57 | - | ||
| 58 | -__aicore__ inline void asc_dcci_entire_out_impl(__gm__ void* dst) | ||
| 59 | -{ | ||
| 60 | - dcci(dst, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_OUT); | ||
| 61 | -} | ||
| 62 | - | ||
| 63 | -__aicore__ inline void asc_dcci_entire_atomic_impl(__gm__ void* dst) | ||
| 64 | -{ | ||
| 65 | - dcci(dst, cache_line_t::ENTIRE_DATA_CACHE, dcci_dst_t::CACHELINE_ATOMIC); | ||
| 66 | -} | ||
| 67 | - | ||
| 68 | 53 | ||
| 69 | 54 | ||
| 70 | 55 | ||
| 71 | 56 | ||
| 72 | 57 | ||
| 73 | 58 | ||
| 74 | - | ||
| @@ -15,4 +15,24 @@ | |||
| 15 | 15 | ||
| 16 | 16 | ||
| 17 | 17 | ||
| 18 | +__aicore__ inline void asc_sync_vec() | ||
| 19 | +{ | ||
| 20 | + asc_sync_vec_impl(); | ||
| 21 | +} | ||
| 22 | + | ||
| 23 | +__aicore__ inline void asc_sync_mte3(int id) | ||
| 24 | +{ | ||
| 25 | + asc_sync_mte3_impl(id); | ||
| 26 | +} | ||
| 27 | + | ||
| 28 | +__aicore__ inline void asc_sync_mte2(int id) | ||
| 29 | +{ | ||
| 30 | + asc_sync_mte2_impl(id); | ||
| 31 | +} | ||
| 32 | + | ||
| 33 | +__aicore__ inline void asc_sync() | ||
| 34 | +{ | ||
| 35 | + asc_sync_impl(); | ||
| 36 | +} | ||
| 37 | + | ||
| 18 | 38 | ||
| @@ -27,6 +27,34 @@ | |||
| 27 | 27 | ||
| 28 | 28 | ||
| 29 | 29 | ||
| 30 | +__aicore__ inline void asc_sync_vec_impl() | ||
| 31 | +{ | ||
| 32 | + pipe_barrier(pipe_t::PIPE_ALL); | ||
| 33 | +} | ||
| 34 | + | ||
| 35 | +__aicore__ inline void asc_sync_mte2_impl(int id) | ||
| 36 | +{ | ||
| 37 | + set_flag(pipe_t::PIPE_MTE2, pipe_t::PIPE_MTE3, static_cast<event_t>(id)); | ||
| 38 | + set_flag(pipe_t::PIPE_MTE2, pipe_t::PIPE_V, static_cast<event_t>(id)); | ||
| 39 | + wait_flag(pipe_t::PIPE_MTE2, pipe_t::PIPE_MTE3, static_cast<event_t>(id)); | ||
| 40 | + wait_flag(pipe_t::PIPE_MTE2, pipe_t::PIPE_V, static_cast<event_t>(id)); | ||
| 41 | + pipe_barrier(pipe_t::PIPE_MTE2); | ||
| 42 | +} | ||
| 43 | + | ||
| 44 | +__aicore__ inline void asc_sync_mte3_impl(int id) | ||
| 45 | +{ | ||
| 46 | + set_flag(pipe_t::PIPE_MTE3, pipe_t::PIPE_MTE2, static_cast<event_t>(id)); | ||
| 47 | + set_flag(pipe_t::PIPE_MTE3, pipe_t::PIPE_V, static_cast<event_t>(id)); | ||
| 48 | + wait_flag(pipe_t::PIPE_MTE3, pipe_t::PIPE_MTE2, static_cast<event_t>(id)); | ||
| 49 | + wait_flag(pipe_t::PIPE_MTE3, pipe_t::PIPE_V, static_cast<event_t>(id)); | ||
| 50 | + pipe_barrier(pipe_t::PIPE_MTE3); | ||
| 51 | +} | ||
| 52 | + | ||
| 53 | +__aicore__ inline void asc_sync_impl() | ||
| 54 | +{ | ||
| 55 | + pipe_barrier(pipe_t::PIPE_ALL); | ||
| 56 | +} | ||
| 57 | + | ||
| 30 | 58 | ||
| 31 | 59 | ||
| 32 | 60 | ||
| @@ -31,34 +31,22 @@ __aicore__ inline void asc_icache_preload(const void* addr, int64_t prefetch_len | |||
| 31 | 31 | ||
| 32 | __aicore__ inline void asc_dcci_single(__gm__ void* dst); | 32 | __aicore__ inline void asc_dcci_single(__gm__ void* dst); |
| 33 | 33 | ||
| 34 | -__aicore__ inline void asc_dcci_entire(__gm__ void* dst); | 34 | +__aicore__ inline void asc_ub_dcci_single(__ubuf__ void* dst); |
| 35 | 35 | ||
| 36 | -__aicore__ inline void asc_dcci_single_all(__gm__ void* dst); | 36 | +#if defined(__NPU_ARCH__) && (__NPU_ARCH__ == 2201) |
| 37 | +__aicore__ inline void asc_dcci_entire_ub(); | ||
| 38 | + | ||
| 37 | 39 | ||
| 38 | -__aicore__ inline void asc_dcci_single_out(__gm__ void* dst); | 40 | +__aicore__ inline void asc_dcci_entire_out(); |
| 39 | 41 | ||
| 40 | -__aicore__ inline void asc_dcci_single_atomic(__gm__ void* dst); | 42 | +__aicore__ inline void asc_dcci_entire_all(); |
| 41 | 43 | ||
| 42 | -__aicore__ inline void asc_dcci_entire_all(__gm__ void* dst); | 44 | +__aicore__ inline void asc_dcci_entire_atomic(); |
| 43 | - | ||
| 44 | -__aicore__ inline void asc_dcci_entire_out(__gm__ void* dst); | ||
| 45 | - | ||
| 46 | -__aicore__ inline void asc_dcci_entire_atomic(__gm__ void* dst); | ||
| 47 | 45 | ||
| 48 | __aicore__ inline int64_t asc_get_icache_preload_status(); | 46 | __aicore__ inline int64_t asc_get_icache_preload_status(); |
| 49 | 47 | ||
| 50 | __aicore__ inline void asc_datacache_preload(__gm__ uint64_t* address, int64_t offset); | 48 | __aicore__ inline void asc_datacache_preload(__gm__ uint64_t* address, int64_t offset); |
| 51 | 49 | ||
| 52 | - | ||
| 53 | - | ||
| 54 | -__aicore__ inline void asc_dcci_single_ub(__gm__ void* dst); | ||
| 55 | - | ||
| 56 | -__aicore__ inline void asc_dcci_entire_ub(__gm__ void* dst); | ||
| 57 | - | ||
| 58 | - | ||
| 59 | - | ||
| 60 | - | ||
| 61 | - | ||
| 62 | 50 | ||
| 63 | 51 | ||
| 64 | 52 | ||
| @@ -14,36 +14,103 @@ | |||
| 14 | 14 | ||
| 15 | 15 | ||
| 16 | 16 | ||
| 17 | -class TestSimdAtomicDcci : public testing::Test { | 17 | +class TestAscCapiSimdDcci : public testing::Test { |
| 18 | protected: | 18 | protected: |
| 19 | void SetUp() {} | 19 | void SetUp() {} |
| 20 | void TearDown() {} | 20 | void TearDown() {} |
| 21 | }; | 21 | }; |
| 22 | 22 | ||
| 23 | namespace { | 23 | namespace { |
| 24 | -void dcci_Stub(__gm__ void* dst) | 24 | +void dcci_single_Stub(__gm__ void* dst, uint64_t entire) |
| 25 | { | 25 | { |
| 26 | EXPECT_EQ((__gm__ void*)3, dst); | 26 | EXPECT_EQ((__gm__ void*)3, dst); |
| 27 | -} | 27 | + EXPECT_EQ(cache_line_t::SINGLE_CACHE_LINE, entire); |
| 28 | } | 28 | } |
| 29 | 29 | ||
| 30 | -#define TEST_DCCI(entire, type) \ | 30 | +void dcci_single_ub_Stub(__ubuf__ void* dst, uint64_t entire) |
| 31 | -TEST_F(TestSimdAtomicDcci, dcci_gm_##entire##_##type##_##void_ptr_uint64_t_Succ) \ | 31 | +{ |
| 32 | -{ \ | 32 | + EXPECT_EQ((__ubuf__ void*)3, dst); |
| 33 | - __gm__ void* dst = (__gm__ void*)3; \ | 33 | + EXPECT_EQ(cache_line_t::SINGLE_CACHE_LINE, entire); |
| 34 | - MOCKER(dcci, void(__gm__ void*, uint64_t, uint64_t)) \ | 34 | +} |
| 35 | - .times(1) \ | 35 | +} // namespace |
| 36 | - .will(invoke(dcci_Stub)); \ | 36 | + |
| 37 | - asc_dcci_##entire##_##type(dst); \ | 37 | +TEST_F(TestAscCapiSimdDcci, dcci_gm_single) |
| 38 | - GlobalMockObject::verify(); \ | 38 | +{ |
| 39 | -} \ | 39 | + __gm__ void* dst = (__gm__ void*)3; |
| 40 | + MOCKER(dcci, void(__gm__ void*, uint64_t)).times(1).will(invoke(dcci_single_Stub)); | ||
| 41 | + asc_dcci_single(dst); | ||
| 42 | + GlobalMockObject ::verify(); | ||
| 43 | +} | ||
| 44 | + | ||
| 45 | +TEST_F(TestAscCapiSimdDcci, dcci_ub_single) | ||
| 46 | +{ | ||
| 47 | + __ubuf__ void* dst = (__ubuf__ void*)3; | ||
| 48 | + MOCKER(dcci, void(__ubuf__ void*, uint64_t)).times(1).will(invoke(dcci_single_ub_Stub)); | ||
| 49 | + asc_ub_dcci_single(dst); | ||
| 50 | + GlobalMockObject ::verify(); | ||
| 51 | +} | ||
| 52 | + | ||
| 53 | +namespace { | ||
| 54 | +void dcci_entire_all_Stub(__gm__ void* dst, uint64_t entire, uint64_t type) | ||
| 55 | +{ | ||
| 56 | + EXPECT_EQ((__gm__ void*)0, dst); | ||
| 57 | + EXPECT_EQ(cache_line_t::ENTIRE_DATA_CACHE, entire); | ||
| 58 | + EXPECT_EQ(dcci_dst_t::CACHELINE_ALL, type); | ||
| 59 | +} | ||
| 60 | +} // namespace | ||
| 40 | 61 | ||
| 41 | 62 | ||
| 42 | -TEST_DCCI(single, all); | 63 | +TEST_F(TestAscCapiSimdDcci, dcci_gm_entire_all) |
| 43 | -TEST_DCCI(single, ub); | 64 | +{ |
| 44 | -TEST_DCCI(single, out); | 65 | + MOCKER(dcci, void(__gm__ void*, uint64_t, uint64_t)).times(1).will(invoke(dcci_entire_all_Stub)); |
| 45 | -TEST_DCCI(single, atomic); | 66 | + asc_dcci_entire_all(); |
| 46 | -TEST_DCCI(entire, all); | 67 | + GlobalMockObject ::verify(); |
| 47 | -TEST_DCCI(entire, ub); | 68 | +} |
| 48 | -TEST_DCCI(entire, out); | 69 | + |
| 49 | -TEST_DCCI(entire, atomic); | 70 | +namespace { |
| 71 | +void dcci_entire_out_Stub(__gm__ void* dst, uint64_t entire, uint64_t type) | ||
| 72 | +{ | ||
| 73 | + EXPECT_EQ((__gm__ void*)0, dst); | ||
| 74 | + EXPECT_EQ(cache_line_t::ENTIRE_DATA_CACHE, entire); | ||
| 75 | + EXPECT_EQ(dcci_dst_t::CACHELINE_OUT, type); | ||
| 76 | +} | ||
| 77 | +} // namespace | ||
| 78 | + | ||
| 79 | +TEST_F(TestAscCapiSimdDcci, dcci_gm_entire_out) | ||
| 80 | +{ | ||
| 81 | + MOCKER(dcci, void(__gm__ void*, uint64_t, uint64_t)).times(1).will(invoke(dcci_entire_out_Stub)); | ||
| 82 | + asc_dcci_entire_out(); | ||
| 83 | + GlobalMockObject ::verify(); | ||
| 84 | +} | ||
| 85 | + | ||
| 86 | +namespace { | ||
| 87 | +void dcci_entire_ub_Stub(__gm__ void* dst, uint64_t entire, uint64_t type) | ||
| 88 | +{ | ||
| 89 | + EXPECT_EQ((__gm__ void*)0, dst); | ||
| 90 | + EXPECT_EQ(cache_line_t::ENTIRE_DATA_CACHE, entire); | ||
| 91 | + EXPECT_EQ(dcci_dst_t::CACHELINE_UB, type); | ||
| 92 | +} | ||
| 93 | +} // namespace | ||
| 94 | + | ||
| 95 | +TEST_F(TestAscCapiSimdDcci, dcci_gm_entire_ub) | ||
| 96 | +{ | ||
| 97 | + MOCKER(dcci, void(__gm__ void*, uint64_t, uint64_t)).times(1).will(invoke(dcci_entire_ub_Stub)); | ||
| 98 | + asc_dcci_entire_ub(); | ||
| 99 | + GlobalMockObject ::verify(); | ||
| 100 | +} | ||
| 101 | + | ||
| 102 | +namespace { | ||
| 103 | +void dcci_entire_atomic_Stub(__gm__ void* dst, uint64_t entire, uint64_t type) | ||
| 104 | +{ | ||
| 105 | + EXPECT_EQ((__gm__ void*)0, dst); | ||
| 106 | + EXPECT_EQ(cache_line_t::ENTIRE_DATA_CACHE, entire); | ||
| 107 | + EXPECT_EQ(dcci_dst_t::CACHELINE_ATOMIC, type); | ||
| 108 | +} | ||
| 109 | +} // namespace | ||
| 110 | + | ||
| 111 | +TEST_F(TestAscCapiSimdDcci, dcci_gm_entire_atomic) | ||
| 112 | +{ | ||
| 113 | + MOCKER(dcci, void(__gm__ void*, uint64_t, uint64_t)).times(1).will(invoke(dcci_entire_atomic_Stub)); | ||
| 114 | + asc_dcci_entire_atomic(); | ||
| 115 | + GlobalMockObject ::verify(); | ||
| 116 | +} | ||
| @@ -13,34 +13,88 @@ | |||
| 13 | 13 | ||
| 14 | 14 | ||
| 15 | 15 | ||
| 16 | -class TestSimdAtomicDcci : public testing::Test { | 16 | +class TestAscCApiSimdDcci : public testing::Test { |
| 17 | protected: | 17 | protected: |
| 18 | void SetUp() {} | 18 | void SetUp() {} |
| 19 | void TearDown() {} | 19 | void TearDown() {} |
| 20 | }; | 20 | }; |
| 21 | 21 | ||
| 22 | namespace { | 22 | namespace { |
| 23 | -void dcci_Stub(__gm__ void* dst) | 23 | +void dcci_single_Stub(__gm__ void* dstPtr, uint64_t entire) |
| 24 | { | 24 | { |
| 25 | - EXPECT_EQ((__gm__ void*)3, dst); | 25 | + EXPECT_EQ((__gm__ void*)3, dstPtr); |
| 26 | -} | 26 | + EXPECT_EQ(cache_line_t::SINGLE_CACHE_LINE, entire); |
| 27 | } | 27 | } |
| 28 | 28 | ||
| 29 | -#define TEST_DCCI(entire, type) \ | 29 | +void dcci_single_ub_Stub(__ubuf__ void* dstPtr, uint64_t entire) |
| 30 | -TEST_F(TestSimdAtomicDcci, dcci_gm_##entire##_##type##_##void_ptr_uint64_t_Succ) \ | 30 | +{ |
| 31 | -{ \ | 31 | + EXPECT_EQ((__ubuf__ void*)3, dstPtr); |
| 32 | - __gm__ void* dst = (__gm__ void*)3; \ | 32 | + EXPECT_EQ(cache_line_t::SINGLE_CACHE_LINE, entire); |
| 33 | - MOCKER(dcci, void(__gm__ void*, uint64_t, uint64_t)) \ | 33 | +} |
| 34 | - .times(1) \ | 34 | +} // namespace |
| 35 | - .will(invoke(dcci_Stub)); \ | 35 | + |
| 36 | - asc_dcci_##entire##_##type(dst); \ | 36 | +TEST_F(TestAscCApiSimdDcci, dcci_gm_single) |
| 37 | - GlobalMockObject::verify(); \ | 37 | +{ |
| 38 | -} \ | 38 | + __gm__ void* dstPtr = (__gm__ void*)3; |
| 39 | + MOCKER(dcci, void(__gm__ void*, uint64_t)).times(1).will(invoke(dcci_single_Stub)); | ||
| 40 | + asc_dcci_single(dstPtr); | ||
| 41 | + GlobalMockObject ::verify(); | ||
| 42 | +} | ||
| 43 | + | ||
| 44 | +TEST_F(TestAscCApiSimdDcci, dcci_ub_single) | ||
| 45 | +{ | ||
| 46 | + __ubuf__ void* dstPtr = (__ubuf__ void*)3; | ||
| 47 | + MOCKER(dcci, void(__ubuf__ void*, uint64_t)).times(1).will(invoke(dcci_single_ub_Stub)); | ||
| 48 | + asc_ub_dcci_single(dstPtr); | ||
| 49 | + GlobalMockObject ::verify(); | ||
| 50 | +} | ||
| 51 | + | ||
| 52 | +namespace { | ||
| 53 | +void dcci_entire_all_stub(__gm__ void* dstPtr, uint64_t entire, uint64_t type) | ||
| 54 | +{ | ||
| 55 | + EXPECT_EQ((__gm__ void*)0, dstPtr); | ||
| 56 | + EXPECT_EQ(cache_line_t::ENTIRE_DATA_CACHE, entire); | ||
| 57 | + EXPECT_EQ(dcci_dst_t::CACHELINE_ALL, type); | ||
| 58 | +} | ||
| 59 | +} // namespace | ||
| 39 | 60 | ||
| 40 | 61 | ||
| 41 | -TEST_DCCI(single, all); | 62 | +TEST_F(TestAscCApiSimdDcci, dcci_gm_entire_all) |
| 42 | -TEST_DCCI(single, out); | 63 | +{ |
| 43 | -TEST_DCCI(single, atomic); | 64 | + MOCKER(dcci, void(__gm__ void*, uint64_t, uint64_t)).times(1).will(invoke(dcci_entire_all_stub)); |
| 44 | -TEST_DCCI(entire, all); | 65 | + asc_dcci_entire_all(); |
| 45 | -TEST_DCCI(entire, out); | 66 | + GlobalMockObject ::verify(); |
| 46 | -TEST_DCCI(entire, atomic); | 67 | +} |
| 68 | + | ||
| 69 | +namespace { | ||
| 70 | +void dcci_entire_out_stub(__gm__ void* dstPtr, uint64_t entire, uint64_t type) | ||
| 71 | +{ | ||
| 72 | + EXPECT_EQ((__gm__ void*)0, dstPtr); | ||
| 73 | + EXPECT_EQ(cache_line_t::ENTIRE_DATA_CACHE, entire); | ||
| 74 | + EXPECT_EQ(dcci_dst_t::CACHELINE_OUT, type); | ||
| 75 | +} | ||
| 76 | +} // namespace | ||
| 77 | + | ||
| 78 | +TEST_F(TestAscCApiSimdDcci, dcci_gm_entire_out) | ||
| 79 | +{ | ||
| 80 | + MOCKER(dcci, void(__gm__ void*, uint64_t, uint64_t)).times(1).will(invoke(dcci_entire_out_stub)); | ||
| 81 | + asc_dcci_entire_out(); | ||
| 82 | + GlobalMockObject ::verify(); | ||
| 83 | +} | ||
| 84 | + | ||
| 85 | + | ||
| 86 | +namespace { | ||
| 87 | +void dcci_entire_atomic_stub(__gm__ void* dstPtr, uint64_t entire, uint64_t type) | ||
| 88 | +{ | ||
| 89 | + EXPECT_EQ((__gm__ void*)0, dstPtr); | ||
| 90 | + EXPECT_EQ(cache_line_t::ENTIRE_DATA_CACHE, entire); | ||
| 91 | + EXPECT_EQ(dcci_dst_t::CACHELINE_ATOMIC, type); | ||
| 92 | +} | ||
| 93 | +} // namespace | ||
| 94 | + | ||
| 95 | +TEST_F(TestAscCApiSimdDcci, dcci_gm_entire_atomic) | ||
| 96 | +{ | ||
| 97 | + MOCKER(dcci, void(__gm__ void*, uint64_t, uint64_t)).times(1).will(invoke(dcci_entire_atomic_stub)); | ||
| 98 | + asc_dcci_entire_atomic(); | ||
| 99 | + GlobalMockObject ::verify(); | ||
| 100 | +} | ||
| @@ -34,10 +34,20 @@ void asc_sync_wait_stub(pipe_t pipe, pipe_t tpipe, event_t id) | |||
| 34 | EXPECT_EQ(event_t::EVENT_ID0, id); | 34 | EXPECT_EQ(event_t::EVENT_ID0, id); |
| 35 | } | 35 | } |
| 36 | 36 | ||
| 37 | -void asc_sync_pipe_stub(pipe_t pipe) | 37 | +void pipe_barrier_all_stub(pipe_t pipe) |
| 38 | { | 38 | { |
| 39 | EXPECT_EQ(pipe_t::PIPE_ALL, pipe); | 39 | EXPECT_EQ(pipe_t::PIPE_ALL, pipe); |
| 40 | } | 40 | } |
| 41 | + | ||
| 42 | +void pipe_barrier_mte2_stub(pipe_t pipe) | ||
| 43 | +{ | ||
| 44 | + EXPECT_EQ(pipe_t::PIPE_MTE2, pipe); | ||
| 45 | +} | ||
| 46 | + | ||
| 47 | +void pipe_barrier_mte3_stub(pipe_t pipe) | ||
| 48 | +{ | ||
| 49 | + EXPECT_EQ(pipe_t::PIPE_MTE3, pipe); | ||
| 50 | +} | ||
| 41 | } | 51 | } |
| 42 | 52 | ||
| 43 | TEST_F(TestSyncInstrCAPI, c_api_sync_notify_Succ) | 53 | TEST_F(TestSyncInstrCAPI, c_api_sync_notify_Succ) |
| @@ -71,8 +81,48 @@ TEST_F(TestSyncInstrCAPI, c_api_sync_pipe_Succ) | |||
| 71 | pipe_t pipe = pipe_t::PIPE_ALL; | 81 | pipe_t pipe = pipe_t::PIPE_ALL; |
| 72 | MOCKER_CPP(pipe_barrier, void(pipe_t)) | 82 | MOCKER_CPP(pipe_barrier, void(pipe_t)) |
| 73 | .times(1) | 83 | .times(1) |
| 74 | - .will(invoke(asc_sync_pipe_stub)); | 84 | + .will(invoke(pipe_barrier_all_stub)); |
| 75 | 85 | ||
| 76 | asc_sync_pipe(pipe); | 86 | asc_sync_pipe(pipe); |
| 77 | GlobalMockObject::verify(); | 87 | GlobalMockObject::verify(); |
| 78 | -} | 88 | +} |
| 89 | + | ||
| 90 | +TEST_F(TestSyncInstrCAPI, asc_sync_vec_Succ) | ||
| 91 | +{ | ||
| 92 | + MOCKER_CPP(pipe_barrier, void(pipe_t)) | ||
| 93 | + .times(1) | ||
| 94 | + .will(invoke(pipe_barrier_all_stub)); | ||
| 95 | + | ||
| 96 | + asc_sync_vec(); | ||
| 97 | + GlobalMockObject::verify(); | ||
| 98 | +} | ||
| 99 | + | ||
| 100 | +TEST_F(TestSyncInstrCAPI, asc_sync_Succ) | ||
| 101 | +{ | ||
| 102 | + MOCKER_CPP(pipe_barrier, void(pipe_t)) | ||
| 103 | + .times(1) | ||
| 104 | + .will(invoke(pipe_barrier_all_stub)); | ||
| 105 | + | ||
| 106 | + asc_sync(); | ||
| 107 | + GlobalMockObject::verify(); | ||
| 108 | +} | ||
| 109 | + | ||
| 110 | +TEST_F(TestSyncInstrCAPI, asc_sync_mte2_Succ) | ||
| 111 | +{ | ||
| 112 | + MOCKER_CPP(pipe_barrier, void(pipe_t)) | ||
| 113 | + .times(1) | ||
| 114 | + .will(invoke(pipe_barrier_mte2_stub)); | ||
| 115 | + | ||
| 116 | + asc_sync_mte2(0); | ||
| 117 | + GlobalMockObject::verify(); | ||
| 118 | +} | ||
| 119 | + | ||
| 120 | +TEST_F(TestSyncInstrCAPI, asc_sync_mte3_Succ) | ||
| 121 | +{ | ||
| 122 | + MOCKER_CPP(pipe_barrier, void(pipe_t)) | ||
| 123 | + .times(1) | ||
| 124 | + .will(invoke(pipe_barrier_mte3_stub)); | ||
| 125 | + | ||
| 126 | + asc_sync_mte3(0); | ||
| 127 | + GlobalMockObject::verify(); | ||
| 128 | +} | ||