Star383
269
代码
代码
Issues12
Pull Requests46
Actions
项目讨论
Wiki
分析
Star383
269
已合并
Add A5 VfSim tileop costmodel tests #1372
Chonghao-Wang创建于 7月23日
Add A5 VfSim tileop costmodel tests #1372
已合并
Chonghao-Wang创建于 7月23日
从Chonghao-Wang/pto-isafeature/a5-vfsim-native-no-loop-fix合入到cann/pto-isamaster
共 78 个文件变更+9522-43
Acmake/a5_vf_mock.cmake+121-0
@@ -0,0 +1,121 @@
1+ 
2+set(_pto_a5_self_dir "${CMAKE_CURRENT_LIST_DIR}")
3+set(_pto_a5_repo_dir "${CMAKE_CURRENT_LIST_DIR}/..")
4+ 
5+function(_pto_a5_build_vfsim)
6+ if(TARGET pto_a5_vfsim)
7+ return()
8+ endif()
9+ set(_vfsim_dir "${_pto_a5_repo_dir}/include/pto/costmodel/a5/VfSim")
10+ find_package(Python3 COMPONENTS Interpreter REQUIRED)
11+ execute_process(
12+ COMMAND ${Python3_EXECUTABLE}
13+ "${_pto_a5_repo_dir}/include/pto/costmodel/a5/formula_costmodel/gen_formula_params_header.py"
14+ WORKING_DIRECTORY "${_pto_a5_repo_dir}"
15+ RESULT_VARIABLE _a5_formula_param_gen_result)
16+ if(NOT _a5_formula_param_gen_result EQUAL 0)
17+ message(FATAL_ERROR "a5_vf_mock: failed to generate a5 formula_params_generated.hpp")
18+ endif()
19+ execute_process(
20+ COMMAND ${Python3_EXECUTABLE} "${_vfsim_dir}/gen_vfsim_params_header.py"
21+ WORKING_DIRECTORY "${_pto_a5_repo_dir}"
22+ RESULT_VARIABLE _vfsim_param_gen_result)
23+ if(NOT _vfsim_param_gen_result EQUAL 0)
24+ message(FATAL_ERROR "a5_vf_mock: failed to generate VfSimParamsGenerated.h")
25+ endif()
26+ add_library(pto_a5_vfsim STATIC
27+ ${_vfsim_dir}/VfInfo.cpp
28+ ${_vfsim_dir}/ParamDB.cpp
29+ ${_vfsim_dir}/ISATraits.cpp
30+ ${_vfsim_dir}/ProgramAnalysis.cpp
31+ ${_vfsim_dir}/ProgramCanonicalization.cpp
32+ ${_vfsim_dir}/ProgramVregLiveRangeNormalization.cpp
33+ ${_vfsim_dir}/ProgramFlatten.cpp
34+ ${_vfsim_dir}/IFU.cpp
35+ ${_vfsim_dir}/IDU.cpp
36+ ${_vfsim_dir}/OOO.cpp
37+ ${_vfsim_dir}/SimulatorRunner.cpp
38+ ${_vfsim_dir}/VfSimCostModel.cpp)
39+ target_include_directories(pto_a5_vfsim PUBLIC "${_pto_a5_repo_dir}/include")
40+ target_compile_features(pto_a5_vfsim PUBLIC cxx_std_20)
41+ target_compile_definitions(pto_a5_vfsim PRIVATE PTO_VFSIM_SOURCE_ROOT="${_vfsim_dir}")
42+endfunction()
43+ 
44+macro(_pto_a5_find_llvm)
45+ if(NOT PTO_A5_LLVM_CONFIG)
46+ if(DEFINED ENV{LLVM_CONFIG} AND EXISTS "$ENV{LLVM_CONFIG}")
47+ set(PTO_A5_LLVM_CONFIG "$ENV{LLVM_CONFIG}")
48+ else()
49+ file(GLOB _pto_a5_bindirs /usr/lib/llvm-*/bin
50+ /usr/local/opt/llvm@*/bin /usr/local/opt/llvm/bin)
51+ find_program(PTO_A5_LLVM_CONFIG
52+ NAMES llvm-config llvm-config-20 llvm-config-19 llvm-config-18
53+ llvm-config-17 llvm-config-16 llvm-config-15 llvm-config-14
54+ PATHS ${_pto_a5_bindirs} /usr/local/bin /usr/bin)
55+ endif()
56+ endif()
57+ if(NOT PTO_A5_LLVM_CONFIG)
58+ message(FATAL_ERROR "a5_vf_mock: llvm-config not found. Mac: brew install llvm@18; "
59+ "Linux: sudo apt install clang-N llvm-N-dev (N>=14), or set LLVM_CONFIG env.")
60+ endif()
61+ execute_process(COMMAND ${PTO_A5_LLVM_CONFIG} --version
62+ OUTPUT_VARIABLE PTO_A5_LLVM_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
63+ execute_process(COMMAND ${PTO_A5_LLVM_CONFIG} --bindir
64+ OUTPUT_VARIABLE PTO_A5_LLVM_BINDIR OUTPUT_STRIP_TRAILING_WHITESPACE)
65+ string(REGEX MATCH "^([0-9]+)" PTO_A5_LLVM_MAJOR "${PTO_A5_LLVM_VERSION}")
66+ if(NOT PTO_A5_CLANGXX OR NOT EXISTS "${PTO_A5_CLANGXX}")
67+ unset(PTO_A5_CLANGXX CACHE)
68+ find_program(PTO_A5_CLANGXX NAMES clang++-${PTO_A5_LLVM_MAJOR} clang++
69+ PATHS ${PTO_A5_LLVM_BINDIR} /usr/bin /usr/local/bin NO_DEFAULT_PATH)
70+ if(NOT PTO_A5_CLANGXX)
71+ find_program(PTO_A5_CLANGXX NAMES clang++-${PTO_A5_LLVM_MAJOR} clang++)
72+ endif()
73+ endif()
74+ if(NOT PTO_A5_CLANGXX)
75+ message(FATAL_ERROR "a5_vf_mock: matching clang++ for LLVM ${PTO_A5_LLVM_MAJOR} not found")
76+ endif()
77+ execute_process(COMMAND ${PTO_A5_LLVM_CONFIG} --cxxflags
78+ OUTPUT_VARIABLE PTO_A5_LLVM_CXXFLAGS OUTPUT_STRIP_TRAILING_WHITESPACE)
79+ separate_arguments(PTO_A5_LLVM_CXXFLAGS NATIVE_COMMAND "${PTO_A5_LLVM_CXXFLAGS}")
80+ message(STATUS "a5_vf_mock: LLVM ${PTO_A5_LLVM_VERSION} (${PTO_A5_LLVM_CONFIG})")
81+endmacro()
82+ 
83+macro(_pto_a5_build_pass)
84+ if(NOT TARGET PtoLoopTracePass)
85+ _pto_a5_find_llvm()
86+ set(_pass_src "${_pto_a5_self_dir}/../include/pto/costmodel/a5/PtoLoopTracePass.cpp")
87+ add_library(PtoLoopTracePass MODULE ${_pass_src})
88+ target_compile_options(PtoLoopTracePass PRIVATE ${PTO_A5_LLVM_CXXFLAGS}
89+ -Wno-unused-command-line-argument -Wno-unknown-warning-option)
90+ if(APPLE)
91+ file(GLOB _sdks "/Library/Developer/CommandLineTools/SDKs/MacOSX*.sdk")
92+ set(_sdk "")
93+ foreach(s IN LISTS _sdks)
94+ if(EXISTS "${s}/usr/include/c++/v1" AND EXISTS "${s}/usr/lib/libc++.tbd")
95+ get_filename_component(_n "${s}" NAME)
96+ string(REGEX MATCH "MacOSX([0-9]+\\.[0-9]+)\\.sdk" _ "${_n}")
97+ if(CMAKE_MATCH_1 AND CMAKE_MATCH_1 VERSION_LESS 16)
98+ set(_sdk "${s}")
99+ endif()
100+ endif()
101+ endforeach()
102+ if(_sdk)
103+ target_compile_options(PtoLoopTracePass PRIVATE -isysroot ${_sdk})
104+ target_link_options(PtoLoopTracePass PRIVATE -undefined dynamic_lookup -nostdlib++
105+ -Wl,-syslibroot,${_sdk})
106+ else()
107+ target_link_options(PtoLoopTracePass PRIVATE -undefined dynamic_lookup -nostdlib++)
108+ endif()
109+ endif()
110+ endif()
111+endmacro()
112+ 
113+function(target_enable_a5_vf_mock target)
114+ _pto_a5_build_pass()
115+ _pto_a5_build_vfsim()
116+ add_dependencies(${target} PtoLoopTracePass)
117+ target_link_libraries(${target} PRIVATE pto_a5_vfsim)
118+ target_compile_options(${target} PRIVATE
119+ -O0 -g -fpass-plugin=$<TARGET_FILE:PtoLoopTracePass>)
120+ message(STATUS "a5_vf_mock: ${target} enabled pass instrumentation (-O0 -g -fpass-plugin)")
121+endfunction()
Minclude/pto/common/cpu_stub.hpp+5-2
@@ -1,5 +1,5 @@
1/**1/**
2-Copyright (c) 2025 Huawei Technologies Co., Ltd.2+Copyright (c) 2026 Huawei Technologies Co., Ltd.
3This program is free software, you can redistribute it and/or modify it under the terms and conditions of3This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4CANN Open Software License Agreement Version 2.0 (the "License").4CANN Open Software License Agreement Version 2.0 (the "License").
5Please refer to the License for details. You may not use this file except in compliance with the License.5Please refer to the License for details. You may not use this file except in compliance with the License.
@@ -7,7 +7,6 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, E
7INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.7INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8See LICENSE in the root of the software repository for the full text of the License.8See LICENSE in the root of the software repository for the full text of the License.
9*/9*/
10- 
11#include <pto/pto-inst.hpp>10#include <pto/pto-inst.hpp>
12#ifndef PTO_CPUSTUB_HPP11#ifndef PTO_CPUSTUB_HPP
13#define PTO_CPUSTUB_HPP12#define PTO_CPUSTUB_HPP
@@ -70,6 +69,7 @@ static inline int aclrtMallocHost(void** p, size_t sz)
70 reinterpret_cast<char*>(dst)[i] = reinterpret_cast<char*>(src)[i]; \69 reinterpret_cast<char*>(dst)[i] = reinterpret_cast<char*>(src)[i]; \
71 }70 }
72 71 
72+#if defined(__CPU_SIM)
73inline int aclrtMemset(void* dst, size_t dstSize, int value, size_t count)73inline int aclrtMemset(void* dst, size_t dstSize, int value, size_t count)
74{74{
75 constexpr int ACL_SUCCESS = 0;75 constexpr int ACL_SUCCESS = 0;
@@ -84,6 +84,7 @@ inline int aclrtMemset(void* dst, size_t dstSize, int value, size_t count)
84 std::fill_n(reinterpret_cast<uint8_t*>(dst), count, static_cast<uint8_t>(value));84 std::fill_n(reinterpret_cast<uint8_t*>(dst), count, static_cast<uint8_t>(value));
85 return ACL_SUCCESS;85 return ACL_SUCCESS;
86}86}
87+#endif
87 88 
88#define aclrtSynchronizeStream(x) (0)89#define aclrtSynchronizeStream(x) (0)
89#define aclrtFree(x) free(x)90#define aclrtFree(x) free(x)
@@ -248,6 +249,7 @@ struct is_event : std::false_type {};
248template <typename... Ts>249template <typename... Ts>
249inline constexpr bool all_events_v = (is_event<Ts>::value && ...);250inline constexpr bool all_events_v = (is_event<Ts>::value && ...);
250 251 
252+#if defined(__CPU_SIM)
251namespace pto {253namespace pto {
252template <SyncCoreType CoreType = SyncCoreType::AIVOnly>254template <SyncCoreType CoreType = SyncCoreType::AIVOnly>
253inline void SYNCALL_IMPL()255inline void SYNCALL_IMPL()
@@ -281,5 +283,6 @@ inline void SYNCALL_SOFT_MIX_IMPL(int32_t* gmWorkspace, int32_t* ubWorkspace, in
281 (void)usedCores;283 (void)usedCores;
282}284}
283} // namespace pto285} // namespace pto
286+#endif // __CPU_SIM
284 287 
285#endif288#endif
Minclude/pto/common/pto_instr_impl.hpp+7-1
@@ -7,7 +7,6 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, E
7INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.7INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8See LICENSE in the root of the software repository for the full text of the License.8See LICENSE in the root of the software repository for the full text of the License.
9*/9*/
10- 
11#ifndef PTO_INSTR_IMPL_HPP10#ifndef PTO_INSTR_IMPL_HPP
12#define PTO_INSTR_IMPL_HPP11#define PTO_INSTR_IMPL_HPP
13 12 
@@ -196,6 +195,12 @@ See LICENSE in the root of the software repository for the full text of the Lice
196#endif195#endif
197 196 
198#ifdef PTO_NPU_ARCH_A5197#ifdef PTO_NPU_ARCH_A5
198+#ifdef __COSTMODEL
199+#include "pto/npu/a5/TAssign.hpp"
200+#include "pto/npu/a5/TSync.hpp"
201+#include "pto/npu/a5/SyncAll.hpp"
202+#include "pto/npu/a5/TAdd.hpp"
203+#else
199#include "pto/npu/a5/TAssign.hpp"204#include "pto/npu/a5/TAssign.hpp"
200#include "pto/npu/a5/TSync.hpp"205#include "pto/npu/a5/TSync.hpp"
201#include "pto/npu/a5/SyncAll.hpp"206#include "pto/npu/a5/SyncAll.hpp"
@@ -320,6 +325,7 @@ See LICENSE in the root of the software repository for the full text of the Lice
320#include "pto/npu/a5/TMulAddDst.hpp"325#include "pto/npu/a5/TMulAddDst.hpp"
321#include "pto/npu/a5/TSubRelu.hpp"326#include "pto/npu/a5/TSubRelu.hpp"
322#include "pto/npu/a5/TFusedMulAddRelu.hpp"327#include "pto/npu/a5/TFusedMulAddRelu.hpp"
328+#endif // __COSTMODEL
323#endif329#endif
324 330 
325#ifdef PTO_NPU_ARCH_A6331#ifdef PTO_NPU_ARCH_A6
Minclude/pto/common/utils.hpp+6-3
@@ -1,5 +1,5 @@
1/**1/**
2-Copyright (c) 2025 Huawei Technologies Co., Ltd.2+Copyright (c) 2026 Huawei Technologies Co., Ltd.
3This program is free software, you can redistribute it and/or modify it under the terms and conditions of3This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4CANN Open Software License Agreement Version 2.0 (the "License").4CANN Open Software License Agreement Version 2.0 (the "License").
5Please refer to the License for details. You may not use this file except in compliance with the License.5Please refer to the License for details. You may not use this file except in compliance with the License.
@@ -7,7 +7,6 @@ THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, E
7INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.7INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8See LICENSE in the root of the software repository for the full text of the License.8See LICENSE in the root of the software repository for the full text of the License.
9*/9*/
10- 
11#ifndef __UIILS_HPP__10#ifndef __UIILS_HPP__
12#define __UIILS_HPP__11#define __UIILS_HPP__
13 12 
@@ -41,6 +40,10 @@ PTO_INTERNAL void SetContinuousMask(unsigned n)
41template <int index>40template <int index>
42PTO_INTERNAL void movemask(uint64_t mask)41PTO_INTERNAL void movemask(uint64_t mask)
43{42{
43+#if defined(__COSTMODEL)
44+ (void)mask;
45+ PTO_STATIC_ASSERT((index <= 1), "movemask: error mask index.");
46+#else
44 if constexpr (index == 0) {47 if constexpr (index == 0) {
45 asm volatile("MOVEMASK MASK[0], %0\n" ::"l"(mask));48 asm volatile("MOVEMASK MASK[0], %0\n" ::"l"(mask));
46 } else if constexpr (index == 1) {49 } else if constexpr (index == 1) {
@@ -48,6 +51,7 @@ PTO_INTERNAL void movemask(uint64_t mask)
48 } else {51 } else {
49 PTO_STATIC_ASSERT((index <= 1), "movemask: error mask index.");52 PTO_STATIC_ASSERT((index <= 1), "movemask: error mask index.");
50 }53 }
54+#endif
51}55}
52 56 
53PTO_INTERNAL void SetVectorCount(uint64_t n) { set_vector_mask(0, n); }57PTO_INTERNAL void SetVectorCount(uint64_t n) { set_vector_mask(0, n); }
@@ -100,7 +104,6 @@ struct B82B16Trait {
100 PTO_INTERNAL static uint64_t TransSize(uint64_t size)104 PTO_INTERNAL static uint64_t TransSize(uint64_t size)
101 {105 {
102 if constexpr (isB8) {106 if constexpr (isB8) {
103- // UB是32B对齐，这是安全的
104 return (size + sizeof(TransType) - 1) / sizeof(TransType);107 return (size + sizeof(TransType) - 1) / sizeof(TransType);
105 } else {108 } else {
106 return size;109 return size;
Ainclude/pto/costmodel/a5/PtoLoopTracePass.cpp+278-0
@@ -0,0 +1,278 @@
1+/**
2+Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+CANN Open Software License Agreement Version 2.0 (the "License").
5+Please refer to the License for details. You may not use this file except in compliance with the License.
6+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+See LICENSE in the root of the software repository for the full text of the License.
9+*/
10+#include "llvm/ADT/APInt.h"
11+#include "llvm/ADT/Hashing.h"
12+#include "llvm/ADT/StringRef.h"
13+#include "llvm/ADT/Twine.h"
14+#include "llvm/Analysis/LoopInfo.h"
15+#include "llvm/IR/Dominators.h"
16+#include "llvm/IR/Constants.h"
17+#include "llvm/IR/DebugInfoMetadata.h"
18+#include "llvm/IR/DebugLoc.h"
19+#include "llvm/IR/DerivedTypes.h"
20+#include "llvm/IR/Function.h"
21+#include "llvm/IR/IRBuilder.h"
22+#include "llvm/IR/Instructions.h"
23+#include "llvm/IR/Module.h"
24+#include "llvm/Passes/PassBuilder.h"
25+#include "llvm/Passes/PassPlugin.h"
26+#include "llvm/Support/Compiler.h"
27+#include "llvm/Support/raw_ostream.h"
28+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
29+#include "llvm/Transforms/Utils/LoopUtils.h" // formDedicatedExitBlocks
30+ 
31+#include <cstring>
32+#include <string>
33+#include <vector>
34+ 
35+using namespace llvm;
36+ 
37+namespace {
38+ 
39+static bool nameStartsWith(StringRef s, const char* p)
40+{
41+ size_t n = std::strlen(p);
42+ return s.size() >= n && std::memcmp(s.data(), p, n) == 0;
43+}
44+static PointerType* i8PtrTy(LLVMContext& C) { return PointerType::getUnqual(Type::getInt8Ty(C)); }
45+ 
46+static FunctionCallee getOrInsertLoopEnter(Module& M)
47+{
48+ LLVMContext& C = M.getContext();
49+ auto* i64 = Type::getInt64Ty(C);
50+ auto* i8p = i8PtrTy(C);
51+ auto* i32 = Type::getInt32Ty(C);
52+ return M.getOrInsertFunction(
53+ "__pto_trace_loop_enter", FunctionType::get(Type::getVoidTy(C), {i64, i8p, i32, i32}, false));
54+}
55+static FunctionCallee getOrInsertLoopIter(Module& M)
56+{
57+ LLVMContext& C = M.getContext();
58+ return M.getOrInsertFunction(
59+ "__pto_trace_loop_iter", FunctionType::get(Type::getVoidTy(C), {Type::getInt64Ty(C)}, false));
60+}
61+static FunctionCallee getOrInsertLoopExit(Module& M)
62+{
63+ LLVMContext& C = M.getContext();
64+ return M.getOrInsertFunction(
65+ "__pto_trace_loop_exit", FunctionType::get(Type::getVoidTy(C), {Type::getInt64Ty(C)}, false));
66+}
67+ 
68+static DebugLoc getLoopDebugLoc(Loop* L)
69+{
70+ if (DebugLoc loc = L->getStartLoc())
71+ return loc;
72+ for (Instruction& I : *L->getHeader())
73+ if (I.getDebugLoc())
74+ return I.getDebugLoc();
75+ return DebugLoc();
76+}
77+ 
78+static uint64_t computeLoopId(Function& F, Loop* L)
79+{
80+ DebugLoc loc = getLoopDebugLoc(L);
81+ std::string key;
82+ raw_string_ostream os(key);
83+ os << F.getName() << "|";
84+ if (loc) {
85+ if (auto* scope = dyn_cast_or_null<DILocalScope>(loc.getScope()))
86+ if (auto* file = scope->getFile())
87+ os << file->getFilename() << "|";
88+ os << loc.getLine() << ":" << loc.getCol();
89+ }
90+ os.flush();
91+ return static_cast<uint64_t>(hash_value(StringRef(key)));
92+}
93+ 
94+struct LoopWork {
95+ uint64_t loopId;
96+ BasicBlock* preheader;
97+ BasicBlock* header;
98+ std::string file;
99+ int line;
100+ int col;
101+ std::vector<BasicBlock*> exitBlocks;
102+};
103+ 
104+static bool isScopeCtorCall(const CallBase* call)
105+{
106+ const Function* callee = call->getCalledFunction();
107+ return callee != nullptr && callee->getName().contains("ScopeSentinelC");
108+}
109+ 
110+static std::vector<const Instruction*> collectScopeEnters(Function& F)
111+{
112+ std::vector<const Instruction*> scopeEnters;
113+ for (BasicBlock& BB : F)
114+ for (Instruction& I : BB)
115+ if (const auto* call = dyn_cast<CallBase>(&I))
116+ if (isScopeCtorCall(call))
117+ scopeEnters.push_back(&I);
118+ return scopeEnters;
119+}
120+ 
121+template <class InScopeFn>
122+static void collectLoopInScope(Loop* L, Function& F, std::vector<LoopWork>& out, InScopeFn&& inScope)
123+{
124+ for (Loop* sub : L->getSubLoops())
125+ collectLoopInScope(sub, F, out, inScope);
126+ 
127+ if (!inScope(L))
128+ return;
129+ 
130+ BasicBlock* pre = L->getLoopPreheader();
131+ DebugLoc loc = getLoopDebugLoc(L);
132+ if (!pre) {
133+ errs() << "[PtoLoopTrace] WARN: loop without preheader skipped (" << F.getName() << ")\n";
134+ return;
135+ }
136+ if (!loc) {
137+ errs() << "[PtoLoopTrace] WARN: loop without DebugLoc skipped (" << F.getName() << ")\n";
138+ return;
139+ }
140+ 
141+ LoopWork w;
142+ w.loopId = computeLoopId(F, L);
143+ w.preheader = pre;
144+ w.header = L->getHeader();
145+ if (auto* scope = dyn_cast_or_null<DILocalScope>(loc.getScope()))
146+ if (auto* file = scope->getFile())
147+ w.file = file->getFilename().str();
148+ w.line = static_cast<int>(loc.getLine());
149+ w.col = static_cast<int>(loc.getCol());
150+ 
151+ if (!L->hasDedicatedExits()) {
152+ errs() << "[PtoLoopTrace] WARN: loop without dedicated exits skipped (" << F.getName() << ")\n";
153+ return;
154+ }
155+ SmallVector<BasicBlock*, 4> exitBlks;
156+ L->getUniqueExitBlocks(exitBlks);
157+ for (BasicBlock* eb : exitBlks)
158+ w.exitBlocks.push_back(eb);
159+ out.push_back(std::move(w));
160+}
161+ 
162+template <class InScopeFn>
163+static bool formDedicatedExitsForScopedLoops(LoopInfo& LI, DominatorTree& DT, InScopeFn&& loopInScope)
164+{
165+ bool cfgChanged = false;
166+ for (Loop* L : LI) {
167+ SmallVector<Loop*, 8> nest;
168+ nest.push_back(L);
169+ for (size_t i = 0; i < nest.size(); ++i)
170+ for (Loop* sub : nest[i]->getSubLoops())
171+ nest.push_back(sub);
172+ for (Loop* cur : nest)
173+ if (loopInScope(cur))
174+ cfgChanged |= formDedicatedExitBlocks(
175+ cur, &DT, &LI,
176+ /*MSSAU=*/nullptr,
177+ /*PreserveLCSSA=*/false);
178+ }
179+ return cfgChanged;
180+}
181+ 
182+static void insertLoopEnterAndIter(LoopWork& w, LLVMContext& C, FunctionCallee enterFn, FunctionCallee iterFn)
183+{
184+ ConstantInt* loopIdC = ConstantInt::get(C, APInt(64, w.loopId));
185+ IRBuilder<> enterBuilder(w.preheader->getTerminator());
186+ Value* filePtr = enterBuilder.CreateGlobalStringPtr(w.file.empty() ? StringRef("") : StringRef(w.file));
187+ enterBuilder.CreateCall(
188+ enterFn, {loopIdC, filePtr, ConstantInt::get(C, APInt(32, static_cast<uint64_t>(w.line))),
189+ ConstantInt::get(C, APInt(32, static_cast<uint64_t>(w.col)))});
190+ 
191+ IRBuilder<> iterBuilder(w.header, w.header->getFirstInsertionPt());
192+ iterBuilder.CreateCall(iterFn, {loopIdC});
193+}
194+ 
195+static void insertLoopExits(LoopWork& w, LLVMContext& C, FunctionCallee exitFn)
196+{
197+ ConstantInt* loopIdC = ConstantInt::get(C, APInt(64, w.loopId));
198+ for (BasicBlock* eb : w.exitBlocks) {
199+ IRBuilder<> b(eb, eb->getFirstInsertionPt());
200+ b.CreateCall(exitFn, {loopIdC});
201+ }
202+}
203+ 
204+struct PtoLoopTracePass : PassInfoMixin<PtoLoopTracePass> {
205+ PreservedAnalyses run(Function& F, FunctionAnalysisManager& FAM)
206+ {
207+ if (F.isDeclaration())
208+ return PreservedAnalyses::all();
209+ 
210+ Module& M = *F.getParent();
211+ LoopInfo& LI = FAM.getResult<LoopAnalysis>(F);
212+ DominatorTree& DT = FAM.getResult<DominatorTreeAnalysis>(F);
213+ 
214+ std::vector<const Instruction*> scopeEnters = collectScopeEnters(F);
215+ 
216+ const bool isDemo = nameStartsWith(F.getName(), "__pto_demo_");
217+ if (scopeEnters.empty() && !isDemo) {
218+ return PreservedAnalyses::all();
219+ }
220+ 
221+ auto loopInScope = [&](Loop* L) -> bool {
222+ if (isDemo)
223+ return true;
224+ BasicBlock* h = L->getHeader();
225+ for (const Instruction* en : scopeEnters)
226+ if (DT.dominates(en, &*h->getFirstInsertionPt()))
227+ return true;
228+ return false;
229+ };
230+ 
231+ bool cfgChanged = formDedicatedExitsForScopedLoops(LI, DT, loopInScope);
232+ if (cfgChanged)
233+ DT.recalculate(F);
234+ 
235+ std::vector<LoopWork> work;
236+ for (Loop* L : LI)
237+ collectLoopInScope(L, F, work, loopInScope);
238+ if (work.empty())
239+ return PreservedAnalyses::all();
240+ 
241+ FunctionCallee enterFn = getOrInsertLoopEnter(M);
242+ FunctionCallee iterFn = getOrInsertLoopIter(M);
243+ FunctionCallee exitFn = getOrInsertLoopExit(M);
244+ LLVMContext& C = M.getContext();
245+ 
246+ bool changed = false;
247+ for (LoopWork& w : work) {
248+ insertLoopEnterAndIter(w, C, enterFn, iterFn);
249+ changed = true;
250+ }
251+ 
252+ for (LoopWork& w : work) {
253+ insertLoopExits(w, C, exitFn);
254+ }
255+ 
256+ return changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
257+ }
258+ 
259+ static bool isRequired() { return true; }
260+};
261+ 
262+} // namespace
263+ 
264+extern "C" LLVM_ATTRIBUTE_WEAK ::llvm::PassPluginLibraryInfo llvmGetPassPluginInfo()
265+{
266+ return {LLVM_PLUGIN_API_VERSION, "PtoLoopTrace", LLVM_VERSION_STRING, [](PassBuilder& PB) {
267+ PB.registerPipelineParsingCallback(
268+ [](StringRef Name, FunctionPassManager& FPM, ArrayRef<PassBuilder::PipelineElement>) {
269+ if (Name != "pto-loop-trace")
270+ return false;
271+ FPM.addPass(PtoLoopTracePass());
272+ return true;
273+ });
274+ PB.registerPipelineStartEPCallback([](ModulePassManager& MPM, OptimizationLevel) {
275+ MPM.addPass(createModuleToFunctionPassAdaptor(PtoLoopTracePass()));
276+ });
277+ }};
278+}
Ainclude/pto/costmodel/a5/VfSim/IDU.cpp+436-0
@@ -0,0 +1,436 @@
1+/**
2+Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+CANN Open Software License Agreement Version 2.0 (the "License").
5+Please refer to the License for details. You may not use this file except in compliance with the License.
6+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+See LICENSE in the root of the software repository for the full text of the License.
9+*/
10+#include "pto/costmodel/a5/VfSim/IDU.h"
11+ 
12+#include "pto/costmodel/a5/VfSim/ISATraits.h"
13+ 
14+#include <algorithm>
15+#include <sstream>
16+#include <stdexcept>
17+#include <utility>
18+ 
19+namespace vfsim {
20+namespace {
21+ 
22+std::string joinInts(const std::vector<int64_t>& values)
23+{
24+ std::ostringstream oss;
25+ for (size_t i = 0; i < values.size(); ++i) {
26+ if (i)
27+ oss << ',';
28+ oss << values[i];
29+ }
30+ return oss.str();
31+}
32+ 
33+} // namespace
34+ 
35+IDU::IDU(
36+ const UarchConfig& uarch, const ParamDB& db, ProgramAnalysis::ParamMap params, std::vector<int64_t> loopBounds,
37+ int64_t totalTopBlocks, std::unordered_map<int, std::vector<int64_t>> topBlockLoopBounds, std::string dtype,
38+ std::unordered_map<std::string, ValueInfo> values)
39+ : db_(db),
40+ dtype_(std::move(dtype)),
41+ analysis_(std::move(params), std::move(values)),
42+ loopBounds_(std::move(loopBounds)),
43+ totalTopBlocks_(totalTopBlocks),
44+ topBlockLoopBounds_(std::move(topBlockLoopBounds))
45+{
46+ windowWidth_ = uarch.iduWindowWidth;
47+ issueWidth_ = uarch.iduIssueWidth;
48+ theoreticalLimitMode_ = false;
49+ theoreticalLimitVloopOnly_ = false;
50+ vfStartupCost_ = db_.isaDefaults().vfStartupCost;
51+ iduDispatchStartAdvance_ = uarch.iduDispatchStartAdvance;
52+ vloopToDispatchDelay_ = uarch.vloopToDispatchDelay;
53+ initialTopBlockVloopStartCycle_ = uarch.initialTopBlockVloopStartCycle;
54+ nestedVloopInitialStartGap_ = uarch.nestedVloopInitialStartGap;
55+ loop1MinFeedbackGap_ = uarch.loop1MinFeedbackGap;
56+ innermostIterDispatchStride_ = uarch.innermostIterDispatchStride;
57+ globalShqPregGate_ = uarch.globalShqPregGate;
58+ initVloopStarts();
59+}
60+ 
61+bool IDU::canAccept() const
62+{
63+ if (theoreticalLimitMode_)
64+ return true;
65+ return static_cast<int64_t>(window_.size()) < windowWidth_;
66+}
67+ 
68+void IDU::accept(const DynamicInst& inst)
69+{
70+ if (theoreticalLimitMode_ || static_cast<int64_t>(window_.size()) < windowWidth_)
71+ window_.push_back(inst);
72+}
73+ 
74+void IDU::setTopBlockVloop(int64_t topBlockId, int64_t startCycle)
75+{
76+ if (topBlockVloopStart_.count(topBlockId))
77+ return;
78+ topBlockVloopStart_[topBlockId] = startCycle;
79+ topBlockBodyOpenTime_[topBlockId] = startCycle + vloopToDispatchDelay_;
80+ vloopTrace_.push_back(VloopTraceRecord{topBlockId, "top_block", {}, startCycle});
81+}
82+ 
83+void IDU::initTopBlockNestedStarts(int64_t topBlockId, int64_t topVloopStart)
84+{
85+ const auto it = topBlockLoopBounds_.find(static_cast<int>(topBlockId));
86+ const std::vector<int64_t>& bounds = it == topBlockLoopBounds_.end() ? loopBounds_ : it->second;
87+ const int64_t depth = static_cast<int64_t>(bounds.size());
88+ if (depth <= 0)
89+ return;
90+ 
91+ setTopBlockVloop(topBlockId, topVloopStart);
92+ if (depth >= 1) {
93+ const std::string key0 = makeKey(topBlockId, "loop0", {});
94+ vloopStart_[key0] = topVloopStart;
95+ bodyOpenTime_[key0] = topVloopStart + vloopToDispatchDelay_;
96+ }
97+ if (depth >= 2 && bounds[0] > 0) {
98+ const std::string key1 = makeKey(topBlockId, "loop1", {0});
99+ vloopStart_[key1] = topVloopStart + nestedVloopInitialStartGap_;
100+ bodyOpenTime_[key1] = vloopStart_[key1] + vloopToDispatchDelay_;
101+ }
102+ if (depth >= 3 && bounds[0] > 0 && bounds[1] > 0) {
103+ const std::string key2 = makeKey(topBlockId, "loop2", {0, 0});
104+ vloopStart_[key2] = topVloopStart + 2 * nestedVloopInitialStartGap_;
105+ bodyOpenTime_[key2] = vloopStart_[key2] + vloopToDispatchDelay_;
106+ }
107+}
108+ 
109+void IDU::initVloopStarts()
110+{
111+ if (totalTopBlocks_ <= 0)
112+ return;
113+ setTopBlockVloop(0, initialTopBlockVloopStartCycle_);
114+ initTopBlockNestedStarts(0, initialTopBlockVloopStartCycle_);
115+}
116+ 
117+std::string IDU::makeKey(int64_t topBlockId, const std::string& loopId, const std::vector<int64_t>& iters) const
118+{
119+ return std::to_string(topBlockId) + "|" + loopId + "|" + joinInts(iters);
120+}
121+ 
122+std::optional<std::string> IDU::normalizeBlockKey(
123+ const std::pair<std::string, std::vector<int64_t>>& raw, int64_t topBlockId) const
124+{
125+ if (raw.first.empty())
126+ return std::nullopt;
127+ return makeKey(topBlockId, raw.first, raw.second);
128+}
129+ 
130+std::optional<std::string> IDU::currentInnerBlockKey(const DynamicInst& inst) const
131+{
132+ const int64_t topBlockId = inst.topBlockId;
133+ const auto& bk = inst.blockKeyByLevel;
134+ if (!bk.empty()) {
135+ return normalizeBlockKey(bk.back(), topBlockId);
136+ }
137+ 
138+ const auto& iterStack = inst.iterStack;
139+ const int64_t depth =
140+ std::min<int64_t>(static_cast<int64_t>(loopBounds_.size()), static_cast<int64_t>(inst.loopDepth));
141+ if (depth <= 0)
142+ return std::nullopt;
143+ if (depth == 1)
144+ return makeKey(topBlockId, "loop0", {});
145+ if (depth == 2 && iterStack.size() >= 1)
146+ return makeKey(topBlockId, "loop1", {iterStack[0]});
147+ if (depth >= 3 && iterStack.size() >= 2)
148+ return makeKey(topBlockId, "loop2", {iterStack[0], iterStack[1]});
149+ return std::nullopt;
150+}
151+ 
152+bool IDU::isLastInstOfTopBlock(const DynamicInst& inst) const
153+{
154+ if (inst.blockEndLevels.empty())
155+ return false;
156+ return inst.isLastInTopBlock;
157+}
158+ 
159+const std::vector<int64_t>& IDU::loopBoundsForTopBlock(int64_t topBlockId) const
160+{
161+ const auto it = topBlockLoopBounds_.find(static_cast<int>(topBlockId));
162+ return it == topBlockLoopBounds_.end() ? loopBounds_ : it->second;
163+}
164+ 
165+bool IDU::hasBlockEndLevel(const DynamicInst& inst, int64_t level) const
166+{
167+ return std::find(inst.blockEndLevels.begin(), inst.blockEndLevels.end(), level) != inst.blockEndLevels.end();
168+}
169+ 
170+void IDU::triggerNextTopBlock(const DynamicInst& inst, int64_t cycle)
171+{
172+ if (!inst.isLastInTopBlock)
173+ return;
174+ const int64_t nextTop = inst.topBlockId + 1;
175+ if (nextTop >= totalTopBlocks_ || topBlockVloopStart_.count(nextTop))
176+ return;
177+ setTopBlockVloop(nextTop, cycle);
178+ initTopBlockNestedStarts(nextTop, cycle);
179+}
180+ 
181+int64_t IDU::lastDispatchOrCycle(const std::string& key, int64_t cycle) const
182+{
183+ const auto it = lastDispatchTime_.find(key);
184+ return it == lastDispatchTime_.end() ? cycle : it->second;
185+}
186+ 
187+void IDU::openLoopBody(const std::string& key, int64_t startCycle)
188+{
189+ vloopStart_[key] = startCycle;
190+ bodyOpenTime_[key] = startCycle + vloopToDispatchDelay_;
191+}
192+ 
193+std::optional<int64_t> IDU::nextLoop1Start(
194+ const DynamicInst& inst, const std::vector<int64_t>& bounds, int64_t cycle) const
195+{
196+ if (!hasBlockEndLevel(inst, 1) || inst.iterStack.empty())
197+ return std::nullopt;
198+ const int64_t topBlockId = inst.topBlockId;
199+ const int64_t i = inst.iterStack[0];
200+ const std::string curKey = makeKey(topBlockId, "loop1", {i});
201+ const int64_t endCy = lastDispatchOrCycle(curKey, cycle);
202+ if (i + 1 >= bounds[0])
203+ return std::nullopt;
204+ const auto startIt = vloopStart_.find(curKey);
205+ const int64_t prevStart = startIt == vloopStart_.end() ? endCy : startIt->second;
206+ return std::max<int64_t>(endCy, prevStart + loop1MinFeedbackGap_);
207+}
208+ 
209+void IDU::triggerDepth2Vloops(const DynamicInst& inst, const std::vector<int64_t>& bounds, int64_t cycle)
210+{
211+ const auto nextStart = nextLoop1Start(inst, bounds, cycle);
212+ if (!nextStart.has_value())
213+ return;
214+ const int64_t topBlockId = inst.topBlockId;
215+ const int64_t i = inst.iterStack[0];
216+ const std::string nextKey = makeKey(topBlockId, "loop1", {i + 1});
217+ openLoopBody(nextKey, *nextStart);
218+ vloopTrace_.push_back(VloopTraceRecord{topBlockId, "loop1", {i + 1}, *nextStart});
219+}
220+ 
221+void IDU::triggerDepth3InnerVloops(const DynamicInst& inst, const std::vector<int64_t>& bounds, int64_t cycle)
222+{
223+ if (!hasBlockEndLevel(inst, 2) || inst.iterStack.size() < 2)
224+ return;
225+ const int64_t topBlockId = inst.topBlockId;
226+ const int64_t i = inst.iterStack[0];
227+ const int64_t j = inst.iterStack[1];
228+ const std::string curKey = makeKey(topBlockId, "loop2", {i, j});
229+ const int64_t endCy = lastDispatchOrCycle(curKey, cycle);
230+ if (j + 1 >= bounds[1])
231+ return;
232+ const std::string nextKey = makeKey(topBlockId, "loop2", {i, j + 1});
233+ openLoopBody(nextKey, endCy);
234+ vloopTrace_.push_back(VloopTraceRecord{topBlockId, "loop2", {i, j + 1}, endCy});
235+}
236+ 
237+void IDU::triggerDepth3OuterVloops(const DynamicInst& inst, const std::vector<int64_t>& bounds, int64_t cycle)
238+{
239+ const auto nextStart = nextLoop1Start(inst, bounds, cycle);
240+ if (!nextStart.has_value())
241+ return;
242+ const int64_t topBlockId = inst.topBlockId;
243+ const int64_t i = inst.iterStack[0];
244+ openLoopBody(makeKey(topBlockId, "loop1", {i + 1}), *nextStart);
245+ vloopTrace_.push_back(VloopTraceRecord{topBlockId, "loop1", {i + 1}, *nextStart});
246+ if (bounds[1] <= 0)
247+ return;
248+ const int64_t childStart = *nextStart + nestedVloopInitialStartGap_;
249+ openLoopBody(makeKey(topBlockId, "loop2", {i + 1, 0}), childStart);
250+ vloopTrace_.push_back(VloopTraceRecord{topBlockId, "loop2", {i + 1, 0}, childStart});
251+}
252+ 
253+void IDU::updateLastDispatch(const DynamicInst& inst, int64_t cycle)
254+{
255+ const int64_t topBlockId = inst.topBlockId;
256+ const auto& bk = inst.blockKeyByLevel;
257+ if (!bk.empty()) {
258+ for (const auto& raw : bk) {
259+ if (auto key = normalizeBlockKey(raw, topBlockId))
260+ lastDispatchTime_[*key] = cycle;
261+ }
262+ return;
263+ }
264+ 
265+ const auto& iterStack = inst.iterStack;
266+ const int64_t depth = static_cast<int64_t>(inst.loopDepth);
267+ if (depth == 1) {
268+ lastDispatchTime_[makeKey(topBlockId, "loop0", {})] = cycle;
269+ } else if (depth == 2 && iterStack.size() >= 1) {
270+ lastDispatchTime_[makeKey(topBlockId, "loop1", {iterStack[0]})] = cycle;
271+ } else if (depth >= 3 && iterStack.size() >= 2) {
272+ lastDispatchTime_[makeKey(topBlockId, "loop2", {iterStack[0], iterStack[1]})] = cycle;
273+ lastDispatchTime_[makeKey(topBlockId, "loop1", {iterStack[0]})] = cycle;
274+ lastDispatchTime_[makeKey(topBlockId, "loop0", {})] = cycle;
275+ }
276+}
277+ 
278+void IDU::triggerNextVloops(const DynamicInst& inst, int64_t cycle)
279+{
280+ const int64_t topBlockId = inst.topBlockId;
281+ const std::vector<int64_t>& bounds = loopBoundsForTopBlock(topBlockId);
282+ const int64_t depth = static_cast<int64_t>(bounds.size());
283+ 
284+ triggerNextTopBlock(inst, cycle);
285+ 
286+ if (depth <= 0 || inst.blockEndLevels.empty())
287+ return;
288+ 
289+ if (depth == 1)
290+ return;
291+ if (depth == 2) {
292+ triggerDepth2Vloops(inst, bounds, cycle);
293+ return;
294+ }
295+ if (depth == 3) {
296+ triggerDepth3InnerVloops(inst, bounds, cycle);
297+ triggerDepth3OuterVloops(inst, bounds, cycle);
298+ }
299+}
300+ 
301+IDU::DispatchResources IDU::makeDispatchResources(const IDUDispatchBudget& budget) const
302+{
303+ constexpr int64_t kUnlimited = 1LL << 60;
304+ if (budget.theoreticalLimitMode) {
305+ return DispatchResources{kUnlimited, kUnlimited, kUnlimited, kUnlimited, static_cast<int64_t>(window_.size())};
306+ }
307+ return DispatchResources{budget.freePreg, budget.freeShqQueue, budget.freeLsq, budget.freeShq, budget.issueBudget};
308+}
309+ 
310+bool IDU::hasInitialDispatchCredit(const DispatchResources& resources) const
311+{
312+ if (resources.shqQueueFree <= 0 && resources.lsqFree <= 0)
313+ return false;
314+ return !globalShqPregGate_ || (resources.credits > 0 && resources.shqFree > 0);
315+}
316+ 
317+bool IDU::isVloopDispatchOpen(const DynamicInst& inst, int64_t cycle, const IDUDispatchBudget& budget)
318+{
319+ const auto topOpenIt = topBlockBodyOpenTime_.find(inst.topBlockId);
320+ if (topOpenIt == topBlockBodyOpenTime_.end() || cycle < topOpenIt->second)
321+ return false;
322+ 
323+ const auto innerKey = currentInnerBlockKey(inst);
324+ if (!budget.theoreticalLimitVloopOnly && innerKey) {
325+ const auto openIt = bodyOpenTime_.find(*innerKey);
326+ if (openIt == bodyOpenTime_.end() || cycle < openIt->second)
327+ return false;
328+ }
329+ if (budget.theoreticalLimitMode || budget.theoreticalLimitVloopOnly || !innerKey) {
330+ return true;
331+ }
332+ 
333+ const int64_t iterId = inst.iterStack.empty() ? 0 : inst.iterStack.back();
334+ if (iterId == 0 && !blockBaseCycle_.count(*innerKey))
335+ blockBaseCycle_[*innerKey] = cycle;
336+ const auto baseIt = blockBaseCycle_.find(*innerKey);
337+ if (baseIt == blockBaseCycle_.end())
338+ return false;
339+ return cycle >= baseIt->second + iterId * innermostIterDispatchStride_;
340+}
341+ 
342+bool IDU::hasQueueResource(const DynamicInst& inst, const std::string& form, const DispatchResources& resources) const
343+{
344+ if (isLoadOp(db_, inst.op, form))
345+ return resources.lsqFree > 0;
346+ if (isStoreOp(db_, inst.op, form))
347+ return resources.lsqFree > 0 && resources.shqFree > 0;
348+ return resources.shqQueueFree > 0 && resources.shqFree > 0;
349+}
350+ 
351+int64_t IDU::countRegisterDst(const DynamicInst& inst) const
352+{
353+ int64_t dstCount = 0;
354+ for (const auto& dst : inst.dst) {
355+ if (analysis_.isVregName(dst))
356+ ++dstCount;
357+ }
358+ return dstCount;
359+}
360+ 
361+void IDU::consumeDispatchResources(
362+ const DynamicInst& inst, const std::string& form, int64_t dstCount, DispatchResources& resources) const
363+{
364+ resources.credits -= dstCount;
365+ if (usesLsq(db_, inst.op, form)) {
366+ --resources.lsqFree;
367+ if (usesSharedShqCredit(db_, inst.op, form))
368+ --resources.shqFree;
369+ } else if (usesShqQueue(db_, inst.op, form)) {
370+ --resources.shqQueueFree;
371+ --resources.shqFree;
372+ }
373+}
374+ 
375+void IDU::recordDispatch(const DynamicInst& inst, int64_t cycle, const DispatchResources& resources)
376+{
377+ window_.pop_front();
378+ dispatchLog_.push_back(IDUDispatchRecord{
379+ cycle,
380+ inst.instId,
381+ inst.op,
382+ inst.dst,
383+ inst.src,
384+ inst.topBlockId,
385+ resources.credits,
386+ resources.shqQueueFree,
387+ resources.lsqFree,
388+ resources.shqFree,
389+ });
390+ updateLastDispatch(inst, cycle);
391+ triggerNextVloops(inst, cycle);
392+}
393+ 
394+std::vector<DynamicInst> IDU::dispatch(int64_t cycle, const IDUDispatchBudget& budget)
395+{
396+ if (window_.empty())
397+ return {};
398+ 
399+ const int64_t dispatchStartGate = std::max<int64_t>(0, vfStartupCost_ - iduDispatchStartAdvance_);
400+ if (cycle < dispatchStartGate)
401+ return {};
402+ 
403+ DispatchResources resources = makeDispatchResources(budget);
404+ 
405+ std::vector<DynamicInst> dispatched;
406+ dispatched.reserve(static_cast<size_t>(std::max<int64_t>(0, resources.issueBudget)));
407+ 
408+ if (!hasInitialDispatchCredit(resources))
409+ return {};
410+ 
411+ for (const auto& inst : window_) {
412+ if (static_cast<int64_t>(dispatched.size()) >= resources.issueBudget)
413+ break;
414+ 
415+ if (!isVloopDispatchOpen(inst, cycle, budget))
416+ break;
417+ 
418+ const std::string& form = inst.form.empty() ? dtype_ : inst.form;
419+ if (!hasQueueResource(inst, form, resources))
420+ break;
421+ 
422+ const int64_t dstCount = countRegisterDst(inst);
423+ if (resources.credits < dstCount)
424+ break;
425+ dispatched.push_back(inst);
426+ consumeDispatchResources(inst, form, dstCount, resources);
427+ }
428+ 
429+ for (const auto& inst : dispatched) {
430+ recordDispatch(inst, cycle, resources);
431+ }
432+ 
433+ return dispatched;
434+}
435+ 
436+} // namespace vfsim
Ainclude/pto/costmodel/a5/VfSim/IDU.h+143-0
@@ -0,0 +1,143 @@
1+/**
2+Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+CANN Open Software License Agreement Version 2.0 (the "License").
5+Please refer to the License for details. You may not use this file except in compliance with the License.
6+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+See LICENSE in the root of the software repository for the full text of the License.
9+*/
10+#ifndef VFSIM_NATIVE_IDU_H
11+#define VFSIM_NATIVE_IDU_H
12+ 
13+#include "pto/costmodel/a5/VfSim/IFU.h"
14+ 
15+#include <deque>
16+#include <optional>
17+#include <string>
18+#include <unordered_map>
19+#include <vector>
20+ 
21+namespace vfsim {
22+ 
23+struct IDUDispatchBudget {
24+ int64_t freePreg = 0;
25+ int64_t freeShqQueue = 0;
26+ int64_t freeLsq = 0;
27+ int64_t freeShq = 0;
28+ int64_t issueBudget = 0;
29+ bool theoreticalLimitMode = false;
30+ bool theoreticalLimitVloopOnly = false;
31+};
32+ 
33+struct IDUDispatchRecord {
34+ int64_t cycle = 0;
35+ int64_t instId = 0;
36+ std::string op;
37+ std::vector<std::string> dst;
38+ std::vector<std::string> src;
39+ int64_t topBlockId = 0;
40+ int64_t vreg = 0;
41+ int64_t shqQueue = 0;
42+ int64_t lsq = 0;
43+ int64_t shq = 0;
44+};
45+ 
46+struct VloopTraceRecord {
47+ int64_t topBlockId = 0;
48+ std::string loopId;
49+ std::vector<int64_t> iter;
50+ int64_t startCycle = 0;
51+};
52+ 
53+class IDU {
54+public:
55+ IDU(const UarchConfig& uarch, const ParamDB& db, ProgramAnalysis::ParamMap params = {},
56+ std::vector<int64_t> loopBounds = {}, int64_t totalTopBlocks = 1,
57+ std::unordered_map<int, std::vector<int64_t>> topBlockLoopBounds = {}, std::string dtype = "fp32",
58+ std::unordered_map<std::string, ValueInfo> values = {});
59+ 
60+ bool empty() const noexcept { return window_.empty(); }
61+ bool canAccept() const;
62+ void accept(const DynamicInst& inst);
63+ 
64+ std::vector<DynamicInst> dispatch(int64_t cycle, const IDUDispatchBudget& budget);
65+ 
66+ const ParamDB& db() const noexcept { return db_; }
67+ const std::vector<IDUDispatchRecord>& dispatchLog() const noexcept { return dispatchLog_; }
68+ const std::vector<VloopTraceRecord>& vloopTrace() const noexcept { return vloopTrace_; }
69+ 
70+private:
71+ struct DispatchResources {
72+ int64_t credits = 0;
73+ int64_t shqQueueFree = 0;
74+ int64_t lsqFree = 0;
75+ int64_t shqFree = 0;
76+ int64_t issueBudget = 0;
77+ };
78+ 
79+ const ParamDB& db_;
80+ std::string dtype_;
81+ int64_t windowWidth_ = 0;
82+ int64_t issueWidth_ = 0;
83+ bool theoreticalLimitMode_ = false;
84+ bool theoreticalLimitVloopOnly_ = false;
85+ int64_t vfStartupCost_ = 0;
86+ int64_t iduDispatchStartAdvance_ = 0;
87+ int64_t vloopToDispatchDelay_ = 4;
88+ int64_t initialTopBlockVloopStartCycle_ = 19;
89+ int64_t nestedVloopInitialStartGap_ = 1;
90+ int64_t loop1MinFeedbackGap_ = 7;
91+ int64_t innermostIterDispatchStride_ = 1;
92+ bool globalShqPregGate_ = false;
93+ 
94+ std::deque<DynamicInst> window_;
95+ ProgramAnalysis analysis_;
96+ std::vector<int64_t> loopBounds_;
97+ int64_t totalTopBlocks_ = 1;
98+ std::unordered_map<int, std::vector<int64_t>> topBlockLoopBounds_;
99+ 
100+ std::unordered_map<int64_t, int64_t> topBlockVloopStart_;
101+ std::unordered_map<int64_t, int64_t> topBlockBodyOpenTime_;
102+ std::unordered_map<std::string, int64_t> vloopStart_;
103+ std::unordered_map<std::string, int64_t> bodyOpenTime_;
104+ std::unordered_map<std::string, int64_t> lastDispatchTime_;
105+ std::unordered_map<std::string, int64_t> blockBaseCycle_;
106+ std::vector<VloopTraceRecord> vloopTrace_;
107+ std::vector<IDUDispatchRecord> dispatchLog_;
108+ 
109+ void initVloopStarts();
110+ void setTopBlockVloop(int64_t topBlockId, int64_t startCycle);
111+ void initTopBlockNestedStarts(int64_t topBlockId, int64_t topVloopStart);
112+ const std::vector<int64_t>& loopBoundsForTopBlock(int64_t topBlockId) const;
113+ bool hasBlockEndLevel(const DynamicInst& inst, int64_t level) const;
114+ void triggerNextTopBlock(const DynamicInst& inst, int64_t cycle);
115+ int64_t lastDispatchOrCycle(const std::string& key, int64_t cycle) const;
116+ void openLoopBody(const std::string& key, int64_t startCycle);
117+ std::optional<int64_t> nextLoop1Start(
118+ const DynamicInst& inst, const std::vector<int64_t>& bounds, int64_t cycle) const;
119+ void triggerDepth2Vloops(const DynamicInst& inst, const std::vector<int64_t>& bounds, int64_t cycle);
120+ void triggerDepth3InnerVloops(const DynamicInst& inst, const std::vector<int64_t>& bounds, int64_t cycle);
121+ void triggerDepth3OuterVloops(const DynamicInst& inst, const std::vector<int64_t>& bounds, int64_t cycle);
122+ 
123+ std::string makeKey(int64_t topBlockId, const std::string& loopId, const std::vector<int64_t>& iters) const;
124+ std::optional<std::string> normalizeBlockKey(
125+ const std::pair<std::string, std::vector<int64_t>>& raw, int64_t topBlockId) const;
126+ std::optional<std::string> currentInnerBlockKey(const DynamicInst& inst) const;
127+ 
128+ void updateLastDispatch(const DynamicInst& inst, int64_t cycle);
129+ void triggerNextVloops(const DynamicInst& inst, int64_t cycle);
130+ DispatchResources makeDispatchResources(const IDUDispatchBudget& budget) const;
131+ bool hasInitialDispatchCredit(const DispatchResources& resources) const;
132+ bool isVloopDispatchOpen(const DynamicInst& inst, int64_t cycle, const IDUDispatchBudget& budget);
133+ bool hasQueueResource(const DynamicInst& inst, const std::string& form, const DispatchResources& resources) const;
134+ int64_t countRegisterDst(const DynamicInst& inst) const;
135+ void consumeDispatchResources(
136+ const DynamicInst& inst, const std::string& form, int64_t dstCount, DispatchResources& resources) const;
137+ void recordDispatch(const DynamicInst& inst, int64_t cycle, const DispatchResources& resources);
138+ bool isLastInstOfTopBlock(const DynamicInst& inst) const;
139+};
140+ 
141+} // namespace vfsim
142+ 
143+#endif // VFSIM_NATIVE_IDU_H
Ainclude/pto/costmodel/a5/VfSim/IFU.cpp+512-0
@@ -0,0 +1,512 @@
1+/**
2+Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+CANN Open Software License Agreement Version 2.0 (the "License").
5+Please refer to the License for details. You may not use this file except in compliance with the License.
6+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+See LICENSE in the root of the software repository for the full text of the License.
9+*/
10+#include "pto/costmodel/a5/VfSim/IFU.h"
11+ 
12+#include <algorithm>
13+#include <stdexcept>
14+#include <utility>
15+ 
16+namespace vfsim {
17+namespace {
18+ 
19+int64_t resolveInt(
20+ const std::string& text, const ProgramAnalysis::ParamMap& params, int64_t defaultValue, int64_t minValue)
21+{
22+ if (text.empty())
23+ return defaultValue;
24+ bool digits = true;
25+ size_t pos = 0;
26+ if (text[0] == '-') {
27+ pos = 1;
28+ digits = text.size() > 1;
29+ }
30+ for (; digits && pos < text.size(); ++pos) {
31+ if (!std::isdigit(static_cast<unsigned char>(text[pos])))
32+ digits = false;
33+ }
34+ if (digits) {
35+ try {
36+ return std::max<int64_t>(minValue, std::stoll(text));
37+ } catch (...) {
38+ return defaultValue;
39+ }
40+ }
41+ auto it = params.find(text);
42+ if (it != params.end())
43+ return std::max<int64_t>(minValue, it->second);
44+ return defaultValue;
45+}
46+ 
47+} // namespace
48+ 
49+IFU::IFU(
50+ const std::vector<LinearProgramNode>& linearNodes, ProgramAnalysis::ParamMap params, const ParamDB* db,
51+ std::unordered_map<int, std::vector<int64_t>> topBlockLoopBounds, int64_t totalTopBlocks, std::string dtype)
52+ : nodes_(linearNodes), analysis_(std::move(params)), dtype_(std::move(dtype))
53+{
54+ db_ = db;
55+ buildIndices();
56+ if (!topBlockLoopBounds.empty())
57+ topBlockLoopBounds_ = std::move(topBlockLoopBounds);
58+ else
59+ topBlockLoopBounds_ = analysis_.inferTopBlockLoopBounds({});
60+ if (totalTopBlocks > 0)
61+ totalTopBlocks_ = totalTopBlocks;
62+}
63+ 
64+bool IFU::isInst(const LinearProgramNode& node) { return node.type == "inst"; }
65+bool IFU::isLoopBegin(const LinearProgramNode& node) { return node.type == "loop_begin"; }
66+bool IFU::isLoopEnd(const LinearProgramNode& node) { return node.type == "loop_end"; }
67+ 
68+bool IFU::containsAnyLoop(const std::vector<LinearProgramNode>& nodes)
69+{
70+ for (const auto& node : nodes) {
71+ if (node.type == "loop_begin")
72+ return true;
73+ }
74+ return false;
75+}
76+ 
77+void IFU::buildIndices()
78+{
79+ const std::vector<int64_t> begins = collectLoopBegins();
80+ markInnermostLoops(begins);
81+ assignTopBlockIds(begins);
82+ cacheLoopBodies(begins);
83+ cacheTopBlockLastInsts();
84+}
85+ 
86+std::vector<int64_t> IFU::collectLoopBegins()
87+{
88+ std::vector<int64_t> stack;
89+ for (int64_t i = 0; i < static_cast<int64_t>(nodes_.size()); ++i) {
90+ const auto& node = nodes_[static_cast<size_t>(i)];
91+ if (isLoopBegin(node)) {
92+ stack.push_back(i);
93+ beginLoopId_[i] = static_cast<int64_t>(beginLoopId_.size());
94+ } else if (isLoopEnd(node)) {
95+ if (stack.empty())
96+ throw std::runtime_error("Unmatched loop_end in linear program");
97+ const int64_t begin = stack.back();
98+ stack.pop_back();
99+ beginToEnd_[begin] = i;
100+ }
101+ }
102+ if (!stack.empty())
103+ throw std::runtime_error("Unmatched loop_begin in linear program");
104+ 
105+ std::vector<int64_t> begins;
106+ begins.reserve(beginToEnd_.size());
107+ for (const auto& [b, _] : beginToEnd_)
108+ begins.push_back(b);
109+ std::sort(begins.begin(), begins.end());
110+ return begins;
111+}
112+ 
113+void IFU::markInnermostLoops(const std::vector<int64_t>& begins)
114+{
115+ for (const auto b : begins) {
116+ const auto e = beginToEnd_.at(b);
117+ bool nested = false;
118+ for (const auto b2 : begins) {
119+ if (b2 == b)
120+ continue;
121+ if (b < b2 && b2 < e) {
122+ nested = true;
123+ break;
124+ }
125+ }
126+ isInnermostBegin_[b] = !nested;
127+ }
128+}
129+ 
130+void IFU::assignTopBlockIds(const std::vector<int64_t>& begins)
131+{
132+ int64_t topBid = 0;
133+ for (const auto b : begins) {
134+ bool enclosed = false;
135+ for (const auto b2 : begins) {
136+ if (b2 == b)
137+ continue;
138+ const auto e2 = beginToEnd_.at(b2);
139+ if (b2 < b && b < e2) {
140+ enclosed = true;
141+ break;
142+ }
143+ }
144+ if (!enclosed)
145+ beginTopBlockId_[b] = topBid++;
146+ }
147+ totalTopBlocks_ = topBid;
148+}
149+ 
150+std::optional<int64_t> IFU::findLastInstIdx(int64_t begin, int64_t end) const
151+{
152+ std::optional<int64_t> lastIdx;
153+ for (int64_t i = begin + 1; i < end; ++i) {
154+ if (isInst(nodes_[static_cast<size_t>(i)]))
155+ lastIdx = i;
156+ }
157+ return lastIdx;
158+}
159+ 
160+void IFU::cacheLoopBodies(const std::vector<int64_t>& begins)
161+{
162+ for (const auto b : begins) {
163+ const auto e = beginToEnd_.at(b);
164+ std::vector<LinearProgramNode> body;
165+ for (int64_t i = b + 1; i < e; ++i) {
166+ if (isInst(nodes_[static_cast<size_t>(i)]))
167+ body.push_back(nodes_[static_cast<size_t>(i)]);
168+ }
169+ if (isInnermostBegin_.at(b))
170+ loopBodyCache_[b] = std::move(body);
171+ 
172+ loopLastInstIdx_[b] = findLastInstIdx(b, e);
173+ }
174+}
175+ 
176+void IFU::cacheTopBlockLastInsts()
177+{
178+ for (const auto& [b, tbid] : beginTopBlockId_) {
179+ const auto e = beginToEnd_.at(b);
180+ topBlockLastInstIdx_[tbid] = findLastInstIdx(b, e);
181+ }
182+}
183+ 
184+bool IFU::done() const { return pc_ >= static_cast<int64_t>(nodes_.size()) && pending_.empty(); }
185+ 
186+std::pair<std::vector<int64_t>, std::vector<int64_t>> IFU::snapshot() const
187+{
188+ std::vector<int64_t> loopIds;
189+ std::vector<int64_t> iterNow;
190+ loopIds.reserve(frames_.size());
191+ iterNow.reserve(frames_.size());
192+ for (const auto& fr : frames_) {
193+ loopIds.push_back(fr.loopId);
194+ iterNow.push_back(fr.iterNow);
195+ }
196+ return {std::move(loopIds), std::move(iterNow)};
197+}
198+ 
199+int64_t IFU::currentTopBlockId() const
200+{
201+ if (!frames_.empty())
202+ return frames_.front().topBlockId;
203+ return 0;
204+}
205+ 
206+std::pair<std::string, std::vector<int64_t>> IFU::makeKey(
207+ int64_t topBlockId, const std::string& loopId, std::vector<int64_t> it) const
208+{
209+ it.insert(it.begin(), topBlockId);
210+ return {loopId, std::move(it)};
211+}
212+ 
213+std::pair<std::string, std::vector<int64_t>> IFU::normalizeBlockKey(
214+ const std::pair<std::string, std::vector<int64_t>>& raw, int64_t topBlockId) const
215+{
216+ if (raw.second.empty())
217+ return raw;
218+ std::vector<int64_t> it = raw.second;
219+ if (!it.empty())
220+ it.insert(it.begin(), topBlockId);
221+ return {raw.first, std::move(it)};
222+}
223+ 
224+std::vector<std::pair<std::string, std::vector<int64_t>>> IFU::buildBlockKeyByLevel(
225+ const std::vector<int64_t>& loopStack, const std::vector<int64_t>& iterStack) const
226+{
227+ std::vector<std::pair<std::string, std::vector<int64_t>>> out;
228+ for (size_t lv = 0; lv < loopStack.size(); ++lv) {
229+ std::vector<int64_t> prefix(iterStack.begin(), iterStack.begin() + static_cast<std::ptrdiff_t>(lv));
230+ out.emplace_back("loop" + std::to_string(lv), std::move(prefix));
231+ }
232+ return out;
233+}
234+ 
235+std::vector<int64_t> IFU::calcBlockEndLevelsNormal() const
236+{
237+ if (frames_.empty())
238+ return {};
239+ 
240+ const int64_t deepest = static_cast<int64_t>(frames_.size()) - 1;
241+ const LoopFrame& deepestFr = frames_.back();
242+ const auto it = loopLastInstIdx_.find(deepestFr.beginIdx);
243+ if (it == loopLastInstIdx_.end() || !it->second.has_value() || pc_ != it->second.value())
244+ return {};
245+ 
246+ std::vector<int64_t> endLevels;
247+ for (int64_t lv = deepest; lv >= 0; --lv) {
248+ if (areFinalLoopIterations(lv, deepest, nullptr, true))
249+ endLevels.push_back(lv);
250+ else
251+ break;
252+ }
253+ return endLevels;
254+}
255+ 
256+bool IFU::areFinalLoopIterations(
257+ int64_t firstLevel, int64_t deepest, const LoopFrame* unrolledFrame, bool isLastSuperIter) const
258+{
259+ for (int64_t kk = firstLevel; kk <= deepest; ++kk) {
260+ const auto& fr = frames_[static_cast<size_t>(kk)];
261+ const bool finalNow = unrolledFrame != nullptr && fr.beginIdx == unrolledFrame->beginIdx ?
262+ isLastSuperIter :
263+ fr.iterNow == fr.itersTotal - 1;
264+ if (!finalNow)
265+ return false;
266+ }
267+ return true;
268+}
269+ 
270+bool IFU::isLastInTopBlockNormal() const
271+{
272+ if (frames_.empty())
273+ return false;
274+ const int64_t tbid = currentTopBlockId();
275+ const auto it = topBlockLastInstIdx_.find(tbid);
276+ if (it == topBlockLastInstIdx_.end() || !it->second.has_value() || pc_ != it->second.value())
277+ return false;
278+ for (const auto& fr : frames_) {
279+ if (fr.iterNow != fr.itersTotal - 1)
280+ return false;
281+ }
282+ return true;
283+}
284+ 
285+DynamicInst IFU::emitNormalInst(const LinearProgramNode& node)
286+{
287+ DynamicInst out;
288+ out.instId = instId_++;
289+ out.type = node.type;
290+ out.op = node.op;
291+ out.form = node.form.empty() ? dtype_ : node.form;
292+ out.src = node.src;
293+ out.dst = node.dst;
294+ 
295+ const auto [loopStack, iterStack] = snapshot();
296+ out.loopStack = loopStack;
297+ out.iterStack = iterStack;
298+ out.loopDepth = static_cast<int64_t>(loopStack.size());
299+ out.inLoop = !loopStack.empty();
300+ out.unrollFactor = 1;
301+ out.lane = -1;
302+ out.topBlockId = currentTopBlockId();
303+ out.isLastInTopBlock = isLastInTopBlockNormal();
304+ out.blockKeyByLevel = buildBlockKeyByLevel(loopStack, iterStack);
305+ out.blockEndLevels = calcBlockEndLevelsNormal();
306+ return out;
307+}
308+ 
309+DynamicInst IFU::emitUnrolledInst(
310+ const LinearProgramNode& node, const LoopFrame& frame, const std::vector<int64_t>& loopStack,
311+ const std::vector<int64_t>& iterStack, int64_t superIter, int64_t lane)
312+{
313+ DynamicInst inst;
314+ inst.instId = instId_++;
315+ inst.type = node.type;
316+ inst.op = node.op;
317+ inst.form = node.form.empty() ? dtype_ : node.form;
318+ inst.src = node.src;
319+ inst.dst = node.dst;
320+ inst.loopStack = loopStack;
321+ if (!iterStack.empty()) {
322+ inst.iterStack = iterStack;
323+ inst.iterStack.back() = superIter;
324+ }
325+ inst.loopDepth = static_cast<int64_t>(loopStack.size());
326+ inst.inLoop = true;
327+ inst.unrollFactor = frame.unroll;
328+ inst.unrollGroup = unrollGroup_;
329+ inst.lane = lane;
330+ inst.origIterBase = frame.iterNow;
331+ for (auto& src : inst.src)
332+ src += "_lane" + std::to_string(lane);
333+ for (auto& dst : inst.dst)
334+ dst += "_lane" + std::to_string(lane);
335+ inst.topBlockId = frame.topBlockId;
336+ inst.isLastInTopBlock = false;
337+ inst.blockKeyByLevel = buildBlockKeyByLevel(loopStack, inst.iterStack);
338+ inst.blockEndLevels.clear();
339+ return inst;
340+}
341+ 
342+std::vector<int64_t> IFU::calcBlockEndLevelsUnrolled(
343+ const std::vector<int64_t>& loopStack, const LoopFrame& frame, bool isLastSuperIter) const
344+{
345+ std::vector<int64_t> endLevels;
346+ if (!isLastSuperIter)
347+ return endLevels;
348+ 
349+ for (int64_t lv = static_cast<int64_t>(loopStack.size()) - 1; lv >= 0; --lv) {
350+ const int64_t deepest = static_cast<int64_t>(frames_.size()) - 1;
351+ if (areFinalLoopIterations(lv, deepest, &frame, isLastSuperIter))
352+ endLevels.push_back(lv);
353+ else
354+ break;
355+ }
356+ return endLevels;
357+}
358+ 
359+bool IFU::isLastUnrolledTopBlock(const LoopFrame& frame) const
360+{
361+ for (const auto& fr : frames_) {
362+ if (fr.beginIdx == frame.beginIdx)
363+ continue;
364+ if (fr.iterNow != fr.itersTotal - 1)
365+ return false;
366+ }
367+ return true;
368+}
369+ 
370+void IFU::markLastPendingUnrolled(
371+ std::vector<DynamicInst>& pending, const std::vector<int64_t>& loopStack, const LoopFrame& frame,
372+ bool isLastSuperIter) const
373+{
374+ if (pending.empty())
375+ return;
376+ pending.back().blockEndLevels = calcBlockEndLevelsUnrolled(loopStack, frame, isLastSuperIter);
377+ if (isLastSuperIter)
378+ pending.back().isLastInTopBlock = isLastUnrolledTopBlock(frame);
379+}
380+ 
381+void IFU::buildPendingUnrolled(LoopFrame& frame)
382+{
383+ const auto it = loopBodyCache_.find(frame.beginIdx);
384+ const std::vector<LinearProgramNode> empty;
385+ const std::vector<LinearProgramNode>& body = it == loopBodyCache_.end() ? empty : it->second;
386+ 
387+ const auto [loopStack, iterStack] = snapshot();
388+ const int64_t U = frame.unroll;
389+ const int64_t origBase = frame.iterNow;
390+ const int64_t superIter = U > 0 ? origBase / U : origBase;
391+ const bool isLastSuperIter = (origBase + U >= frame.itersTotal);
392+ 
393+ std::vector<DynamicInst> pending;
394+ pending.reserve(body.size() * static_cast<size_t>(std::max<int64_t>(1, U)));
395+ 
396+ for (const auto& ins : body) {
397+ for (int64_t lane = 0; lane < U; ++lane) {
398+ pending.push_back(emitUnrolledInst(ins, frame, loopStack, iterStack, superIter, lane));
399+ }
400+ }
401+ 
402+ markLastPendingUnrolled(pending, loopStack, frame, isLastSuperIter);
403+ 
404+ ++unrollGroup_;
405+ for (auto& inst : pending)
406+ pending_.push_back(std::move(inst));
407+ frame.iterNow += U;
408+}
409+ 
410+std::optional<DynamicInst> IFU::popPending()
411+{
412+ if (pending_.empty())
413+ return std::nullopt;
414+ DynamicInst out = std::move(pending_.front());
415+ pending_.pop_front();
416+ return out;
417+}
418+ 
419+void IFU::enterLoop(const LinearProgramNode& node)
420+{
421+ const int64_t iters = resolveInt(node.itersRaw, analysis_.params(), 1, 0);
422+ const int64_t end = beginToEnd_.at(pc_);
423+ const int64_t loopId = beginLoopId_.at(pc_);
424+ const bool isInnermost = isInnermostBegin_.at(pc_);
425+ const int64_t unroll = resolveInt(node.unrollRaw, analysis_.params(), 1, 1);
426+ if (iters <= 0) {
427+ pc_ = end + 1;
428+ return;
429+ }
430+ 
431+ const int64_t topBlockId = frames_.empty() ? beginTopBlockId_.at(pc_) : frames_.front().topBlockId;
432+ if (isInnermost && unroll > 1 && iters % unroll != 0)
433+ throw std::runtime_error("Invalid unroll: iters not divisible by unroll");
434+ 
435+ frames_.push_back(
436+ LoopFrame{pc_, end, loopId, iters, 0, isInnermost, (isInnermost && unroll > 1) ? unroll : 1, topBlockId});
437+ pc_ = isInnermost && unroll > 1 ? end : pc_ + 1;
438+}
439+ 
440+std::optional<DynamicInst> IFU::handleLoopEnd()
441+{
442+ if (frames_.empty())
443+ throw std::runtime_error("loop_end encountered with empty runtime stack");
444+ LoopFrame& top = frames_.back();
445+ if (top.endIdx != pc_)
446+ throw std::runtime_error("loop_end mismatch with runtime top frame");
447+ 
448+ if (top.isInnermost && top.unroll > 1) {
449+ if (top.iterNow < top.itersTotal) {
450+ buildPendingUnrolled(top);
451+ return popPending();
452+ }
453+ frames_.pop_back();
454+ ++pc_;
455+ return std::nullopt;
456+ }
457+ 
458+ if (top.iterNow + 1 < top.itersTotal) {
459+ ++top.iterNow;
460+ pc_ = top.beginIdx + 1;
461+ return std::nullopt;
462+ }
463+ 
464+ frames_.pop_back();
465+ ++pc_;
466+ return std::nullopt;
467+}
468+ 
469+std::optional<DynamicInst> IFU::nextInst()
470+{
471+ if (auto out = popPending())
472+ return out;
473+ 
474+ while (pc_ < static_cast<int64_t>(nodes_.size())) {
475+ const auto& n = nodes_[static_cast<size_t>(pc_)];
476+ if (n.type == "loop_begin") {
477+ enterLoop(n);
478+ continue;
479+ }
480+ 
481+ if (n.type == "loop_end") {
482+ if (auto out = handleLoopEnd())
483+ return out;
484+ continue;
485+ }
486+ 
487+ if (n.type != "inst") {
488+ ++pc_;
489+ continue;
490+ }
491+ 
492+ DynamicInst out = emitNormalInst(n);
493+ ++pc_;
494+ return out;
495+ }
496+ 
497+ return std::nullopt;
498+}
499+ 
500+std::vector<DynamicInst> IFU::take(int64_t n)
501+{
502+ std::vector<DynamicInst> out;
503+ for (int64_t i = 0; i < std::max<int64_t>(0, n); ++i) {
504+ auto inst = nextInst();
505+ if (!inst.has_value())
506+ break;
507+ out.push_back(std::move(*inst));
508+ }
509+ return out;
510+}
511+ 
512+} // namespace vfsim
Ainclude/pto/costmodel/a5/VfSim/IFU.h+145-0
@@ -0,0 +1,145 @@
1+/**
2+Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+CANN Open Software License Agreement Version 2.0 (the "License").
5+Please refer to the License for details. You may not use this file except in compliance with the License.
6+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+See LICENSE in the root of the software repository for the full text of the License.
9+*/
10+#ifndef VFSIM_NATIVE_IFU_H
11+#define VFSIM_NATIVE_IFU_H
12+ 
13+#include "pto/costmodel/a5/VfSim/ParamDB.h"
14+#include "pto/costmodel/a5/VfSim/ProgramFlatten.h"
15+ 
16+#include <deque>
17+#include <optional>
18+#include <string>
19+#include <utility>
20+#include <vector>
21+ 
22+namespace vfsim {
23+ 
24+struct DynamicInst {
25+ int64_t instId = 0;
26+ std::string type = "inst";
27+ std::string op;
28+ std::string form;
29+ std::vector<std::string> src;
30+ std::vector<std::string> dst;
31+ 
32+ std::vector<int64_t> loopStack;
33+ std::vector<int64_t> iterStack;
34+ int64_t loopDepth = 0;
35+ bool inLoop = false;
36+ 
37+ int64_t unrollFactor = 1;
38+ int64_t lane = -1;
39+ int64_t unrollGroup = 0;
40+ int64_t origIterBase = 0;
41+ int64_t topBlockId = 0;
42+ bool isLastInTopBlock = false;
43+ std::vector<std::pair<std::string, std::vector<int64_t>>> blockKeyByLevel;
44+ std::vector<int64_t> blockEndLevels;
45+};
46+ 
47+class IFU {
48+public:
49+ IFU(const std::vector<LinearProgramNode>& linearNodes, ProgramAnalysis::ParamMap params = {},
50+ const ParamDB* db = nullptr, std::unordered_map<int, std::vector<int64_t>> topBlockLoopBounds = {},
51+ int64_t totalTopBlocks = 0, std::string dtype = "fp32");
52+ 
53+ bool done() const;
54+ std::optional<DynamicInst> nextInst();
55+ std::vector<DynamicInst> take(int64_t n);
56+ 
57+ int totalTopBlocks() const noexcept { return totalTopBlocks_; }
58+ const std::unordered_map<int, std::vector<int64_t>>& topBlockLoopBounds() const noexcept
59+ {
60+ return topBlockLoopBounds_;
61+ }
62+ const std::vector<std::pair<int64_t, std::vector<int64_t>>>& vloopTrace() const noexcept { return vloopTrace_; }
63+ 
64+private:
65+ struct LoopFrame {
66+ int64_t beginIdx = 0;
67+ int64_t endIdx = 0;
68+ int64_t loopId = 0;
69+ int64_t itersTotal = 0;
70+ int64_t iterNow = 0;
71+ bool isInnermost = false;
72+ int64_t unroll = 1;
73+ int64_t topBlockId = 0;
74+ };
75+ 
76+ std::vector<LinearProgramNode> nodes_;
77+ ProgramAnalysis analysis_;
78+ const ParamDB* db_ = nullptr;
79+ std::string dtype_;
80+ 
81+ std::unordered_map<int64_t, int64_t> beginToEnd_;
82+ std::unordered_map<int64_t, int64_t> beginLoopId_;
83+ std::unordered_map<int64_t, bool> isInnermostBegin_;
84+ std::unordered_map<int64_t, int64_t> beginTopBlockId_;
85+ std::unordered_map<int64_t, std::vector<LinearProgramNode>> loopBodyCache_;
86+ std::unordered_map<int64_t, std::optional<int64_t>> loopLastInstIdx_;
87+ std::unordered_map<int64_t, std::optional<int64_t>> topBlockLastInstIdx_;
88+ int64_t totalTopBlocks_ = 0;
89+ 
90+ int64_t pc_ = 0;
91+ int64_t instId_ = 0;
92+ int64_t unrollGroup_ = 0;
93+ std::deque<DynamicInst> pending_;
94+ std::vector<LoopFrame> frames_;
95+ std::vector<std::pair<int64_t, std::vector<int64_t>>> vloopTrace_;
96+ std::unordered_map<int, std::vector<int64_t>> topBlockLoopBounds_;
97+ 
98+ void buildIndices();
99+ std::vector<int64_t> collectLoopBegins();
100+ void markInnermostLoops(const std::vector<int64_t>& begins);
101+ void assignTopBlockIds(const std::vector<int64_t>& begins);
102+ std::optional<int64_t> findLastInstIdx(int64_t begin, int64_t end) const;
103+ void cacheLoopBodies(const std::vector<int64_t>& begins);
104+ void cacheTopBlockLastInsts();
105+ static bool containsAnyLoop(const std::vector<LinearProgramNode>& nodes);
106+ static bool isInst(const LinearProgramNode& node);
107+ static bool isLoopBegin(const LinearProgramNode& node);
108+ static bool isLoopEnd(const LinearProgramNode& node);
109+ 
110+ std::pair<std::vector<int64_t>, std::vector<int64_t>> snapshot() const;
111+ int64_t currentTopBlockId() const;
112+ std::pair<std::string, std::vector<int64_t>> normalizeBlockKey(
113+ const std::pair<std::string, std::vector<int64_t>>& raw, int64_t topBlockId) const;
114+ std::pair<std::string, std::vector<int64_t>> makeKey(
115+ int64_t topBlockId, const std::string& loopId, std::vector<int64_t> it) const;
116+ std::pair<std::vector<int64_t>, std::vector<int64_t>> currentBlockKeyPair(const DynamicInst& inst) const;
117+ 
118+ std::vector<std::pair<std::string, std::vector<int64_t>>> buildBlockKeyByLevel(
119+ const std::vector<int64_t>& loopStack, const std::vector<int64_t>& iterStack) const;
120+ std::vector<int64_t> calcBlockEndLevelsNormal() const;
121+ bool areFinalLoopIterations(
122+ int64_t firstLevel, int64_t deepest, const LoopFrame* unrolledFrame, bool isLastSuperIter) const;
123+ bool isLastInTopBlockNormal() const;
124+ DynamicInst emitNormalInst(const LinearProgramNode& node);
125+ DynamicInst emitUnrolledInst(
126+ const LinearProgramNode& node, const LoopFrame& frame, const std::vector<int64_t>& loopStack,
127+ const std::vector<int64_t>& iterStack, int64_t superIter, int64_t lane);
128+ std::vector<int64_t> calcBlockEndLevelsUnrolled(
129+ const std::vector<int64_t>& loopStack, const LoopFrame& frame, bool isLastSuperIter) const;
130+ bool isLastUnrolledTopBlock(const LoopFrame& frame) const;
131+ void markLastPendingUnrolled(
132+ std::vector<DynamicInst>& pending, const std::vector<int64_t>& loopStack, const LoopFrame& frame,
133+ bool isLastSuperIter) const;
134+ void buildPendingUnrolled(LoopFrame& frame);
135+ std::optional<DynamicInst> popPending();
136+ void enterLoop(const LinearProgramNode& node);
137+ std::optional<DynamicInst> handleLoopEnd();
138+ 
139+ void updateLastDispatch(const DynamicInst& inst, int64_t cycle);
140+ void triggerNextVloops(const DynamicInst& inst, int64_t cycle);
141+};
142+ 
143+} // namespace vfsim
144+ 
145+#endif // VFSIM_NATIVE_IFU_H
Ainclude/pto/costmodel/a5/VfSim/ISATraits.cpp+95-0
@@ -0,0 +1,95 @@
1+/**
2+Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+CANN Open Software License Agreement Version 2.0 (the "License").
5+Please refer to the License for details. You may not use this file except in compliance with the License.
6+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+See LICENSE in the root of the software repository for the full text of the License.
9+*/
10+#include "pto/costmodel/a5/VfSim/ISATraits.h"
11+ 
12+#include <cctype>
13+#include <algorithm>
14+ 
15+namespace vfsim {
16+namespace {
17+ 
18+std::string canonicalOp(std::string op)
19+{
20+ std::transform(
21+ op.begin(), op.end(), op.begin(), [](unsigned char c) { return static_cast<char>(std::toupper(c)); });
22+ return op;
23+}
24+ 
25+OpClass opClassFromString(const std::string& text)
26+{
27+ const std::string canon = canonicalOp(text);
28+ if (canon == "LOAD")
29+ return OpClass::Load;
30+ if (canon == "STORE")
31+ return OpClass::Store;
32+ if (canon == "COMPUTE")
33+ return OpClass::Compute;
34+ return OpClass::Unknown;
35+}
36+ 
37+OpClass opClassFromNameFallback(const std::string& op)
38+{
39+ const std::string canon = canonicalOp(op);
40+ if (canon == "VLDS" || canon == "VLD")
41+ return OpClass::Load;
42+ if (canon == "VSTS" || canon == "VST" || canon == "VSTUS" || canon == "VSTAS")
43+ return OpClass::Store;
44+ return OpClass::Compute;
45+}
46+ 
47+} // namespace
48+ 
49+OpClass getOpClass(const ParamDB& db, const std::string& op, const std::string& dtype)
50+{
51+ const std::string canonOp = canonicalOp(op);
52+ if (db.hasInst(canonOp, dtype)) {
53+ const InstConfig& cfg = db.inst(canonOp, dtype);
54+ const OpClass opClass = opClassFromString(cfg.opClass);
55+ if (opClass != OpClass::Unknown)
56+ return opClass;
57+ if (!cfg.exu.empty() || !cfg.dispatchExu.empty())
58+ return OpClass::Compute;
59+ }
60+ return opClassFromNameFallback(canonOp);
61+}
62+ 
63+bool isLoadOp(const ParamDB& db, const std::string& op, const std::string& dtype)
64+{
65+ return getOpClass(db, op, dtype) == OpClass::Load;
66+}
67+ 
68+bool isStoreOp(const ParamDB& db, const std::string& op, const std::string& dtype)
69+{
70+ return getOpClass(db, op, dtype) == OpClass::Store;
71+}
72+ 
73+bool isComputeOp(const ParamDB& db, const std::string& op, const std::string& dtype)
74+{
75+ return getOpClass(db, op, dtype) == OpClass::Compute;
76+}
77+ 
78+bool usesLsq(const ParamDB& db, const std::string& op, const std::string& dtype)
79+{
80+ const OpClass cls = getOpClass(db, op, dtype);
81+ return cls == OpClass::Load || cls == OpClass::Store;
82+}
83+ 
84+bool usesShqQueue(const ParamDB& db, const std::string& op, const std::string& dtype)
85+{
86+ return isComputeOp(db, op, dtype);
87+}
88+ 
89+bool usesSharedShqCredit(const ParamDB& db, const std::string& op, const std::string& dtype)
90+{
91+ const OpClass cls = getOpClass(db, op, dtype);
92+ return cls == OpClass::Compute || cls == OpClass::Store;
93+}
94+ 
95+} // namespace vfsim
Ainclude/pto/costmodel/a5/VfSim/ISATraits.h+36-0
@@ -0,0 +1,36 @@
1+/**
2+Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+CANN Open Software License Agreement Version 2.0 (the "License").
5+Please refer to the License for details. You may not use this file except in compliance with the License.
6+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+See LICENSE in the root of the software repository for the full text of the License.
9+*/
10+#ifndef VFSIM_NATIVE_ISA_TRAITS_H
11+#define VFSIM_NATIVE_ISA_TRAITS_H
12+ 
13+#include "pto/costmodel/a5/VfSim/ParamDB.h"
14+ 
15+#include <string>
16+ 
17+namespace vfsim {
18+ 
19+enum class OpClass {
20+ Load,
21+ Store,
22+ Compute,
23+ Unknown,
24+};
25+ 
26+OpClass getOpClass(const ParamDB& db, const std::string& op, const std::string& dtype = "fp32");
27+bool isLoadOp(const ParamDB& db, const std::string& op, const std::string& dtype = "fp32");
28+bool isStoreOp(const ParamDB& db, const std::string& op, const std::string& dtype = "fp32");
29+bool isComputeOp(const ParamDB& db, const std::string& op, const std::string& dtype = "fp32");
30+bool usesLsq(const ParamDB& db, const std::string& op, const std::string& dtype = "fp32");
31+bool usesShqQueue(const ParamDB& db, const std::string& op, const std::string& dtype = "fp32");
32+bool usesSharedShqCredit(const ParamDB& db, const std::string& op, const std::string& dtype = "fp32");
33+ 
34+} // namespace vfsim
35+ 
36+#endif // VFSIM_NATIVE_ISA_TRAITS_H
Ainclude/pto/costmodel/a5/VfSim/JsonDumpUtils.h+84-0
@@ -0,0 +1,84 @@
1+/**
2+Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+CANN Open Software License Agreement Version 2.0 (the "License").
5+Please refer to the License for details. You may not use this file except in compliance with the License.
6+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+See LICENSE in the root of the software repository for the full text of the License.
9+*/
10+#ifndef VFSIM_NATIVE_JSON_DUMP_UTILS_H
11+#define VFSIM_NATIVE_JSON_DUMP_UTILS_H
12+ 
13+#include <optional>
14+#include <sstream>
15+#include <string>
16+#include <vector>
17+ 
18+namespace vfsim {
19+ 
20+inline std::string jsonEscape(const std::string& text)
21+{
22+ std::string out;
23+ out.reserve(text.size() + 8);
24+ for (char c : text) {
25+ switch (c) {
26+ case '\\':
27+ out += "\\\\";
28+ break;
29+ case '"':
30+ out += "\\\"";
31+ break;
32+ case '\n':
33+ out += "\\n";
34+ break;
35+ case '\r':
36+ out += "\\r";
37+ break;
38+ case '\t':
39+ out += "\\t";
40+ break;
41+ default:
42+ out.push_back(c);
43+ break;
44+ }
45+ }
46+ return out;
47+}
48+ 
49+template <typename T>
50+inline void appendJsonValue(std::ostringstream& oss, const T& value)
51+{
52+ oss << value;
53+}
54+ 
55+inline void appendJsonValue(std::ostringstream& oss, const std::string& value)
56+{
57+ oss << '"' << jsonEscape(value) << '"';
58+}
59+ 
60+inline void appendJsonValue(std::ostringstream& oss, const std::optional<std::string>& value)
61+{
62+ if (value.has_value())
63+ appendJsonValue(oss, *value);
64+ else
65+ oss << "null";
66+}
67+ 
68+template <typename T>
69+inline std::string joinJsonArray(const std::vector<T>& values)
70+{
71+ std::ostringstream oss;
72+ oss << "[";
73+ for (size_t i = 0; i < values.size(); ++i) {
74+ if (i)
75+ oss << ", ";
76+ appendJsonValue(oss, values[i]);
77+ }
78+ oss << "]";
79+ return oss.str();
80+}
81+ 
82+} // namespace vfsim
83+ 
84+#endif // VFSIM_NATIVE_JSON_DUMP_UTILS_H
Ainclude/pto/costmodel/a5/VfSim/OOO.cpp+1060-0
@@ -0,0 +1,1060 @@
1+/**
2+Copyright (c) 2026 Huawei Technologies Co., Ltd.
3+This program is free software, you can redistribute it and/or modify it under the terms and conditions of
4+CANN Open Software License Agreement Version 2.0 (the "License").
5+Please refer to the License for details. You may not use this file except in compliance with the License.
6+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
7+INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
8+See LICENSE in the root of the software repository for the full text of the License.
9+*/
10+#include "pto/costmodel/a5/VfSim/OOO.h"
11+ 
12+#include "pto/costmodel/a5/VfSim/ISATraits.h"
13+#include "pto/costmodel/a5/VfSim/JsonDumpUtils.h"
14+ 
15+#include <algorithm>
16+#include <fstream>
17+#include <stdexcept>
18+#include <utility>
19+ 
20+namespace vfsim {
21+namespace {
22+ 
23+bool isIntermediateMemName(const std::string& name)
24+{
25+ std::string lower = name;
26+ std::transform(
27+ lower.begin(), lower.end(), lower.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
28+ return lower.rfind("mem_inter", 0) == 0;
29+}
30+ 
31+} // namespace
32+ 
33+OoOCore::OoOCore(
34+ const UarchConfig& uarch, const ParamDB& db, std::string dtype,
35+ const std::unordered_map<std::string, ValueInfo>& values)
36+ : db_(db), dtype_(std::move(dtype)), valueStorage_(values)
37+{
38+ theoreticalLimitMode_ = false;
39+ enableIsuQueueModel_ = uarch.enableIsuQueueModel;
40+ loadPorts_ = static_cast<int>(uarch.loadPorts);
41+ issuePorts_ = static_cast<int>(uarch.issuePorts);
42+ storePorts_ = static_cast<int>(uarch.storePorts);
43+ shqDepth_ = static_cast<int>(uarch.shqDepth);
44+ lsqDepth_ = static_cast<int>(uarch.ldqWidth ? uarch.ldqWidth : 24);
45+ pregNum_ = static_cast<int>(uarch.vregNum ? uarch.vregNum : 68);
46+ vfStartupCost_ = static_cast<int>(db_.isaDefaults().vfStartupCost);
47+ vfDrainCost_ = static_cast<int>(db_.isaDefaults().vfDrainCost);
48+ freelist_.clear();
49+ for (int i = 0; i < pregNum_; ++i)
50+ freelist_.push_back("p" + std::to_string(i));
51+ visiblePregFree_ = pregNum_;
52+ lastIssueCycleALU_.assign(issuePorts_, -1000000000);
53+ lastIssueCycleSFU_.assign(issuePorts_, -1000000000);
54+ lastOpALU_.assign(issuePorts_, "");
55+ lastFormALU_.assign(issuePorts_, "");
56+ lastOpSFU_.assign(issuePorts_, "");
57+ lastFormSFU_.assign(issuePorts_, "");
58+ lastIssueCycleExu_.assign(issuePorts_, -1000000000);
59+ lastOpExu_.assign(issuePorts_, "");
60+ lastFormExu_.assign(issuePorts_, "");
61+ exqInflight_.assign(issuePorts_, 0);
62+ loadDoneLatency_ = static_cast<int>(uarch.loadDoneLatency ? uarch.loadDoneLatency : 9);
63+ oooToShqDelay_ = static_cast<int>(uarch.oooToShqDelay ? uarch.oooToShqDelay : 1);
64+ oooToLsqDelay_ = static_cast<int>(uarch.oooToLsqDelay ? uarch.oooToLsqDelay : 1);
65+ exqRecvDelay_ = static_cast<int>(uarch.exqRecvDelay ? uarch.exqRecvDelay : 1);
66+ memBarStrong_ = uarch.memBarMode == "strong";
67+ enforceSameCycleSrcHazard_ = uarch.enforceSameCycleSrcHazard;
68+ enableExqGreedyBalance_ = false;
69+ enableShqCreditModel_ = uarch.enableShqCreditModel;
70+ enableCreditVisibilityDelay_ = uarch.enableCreditVisibilityDelay;
71+ enableCrossFuIi_ = uarch.enableCrossFuIi;
72+ exqCapacityCountsInflight_ = uarch.exqCapacityCountsInflight;
73+ exqDepth_ = static_cast<int>(uarch.exqDepth ? uarch.exqDepth : 26);
74+ shqToExqPortPerCycle_ = static_cast<int>(uarch.shqToExqPortPerCycle ? uarch.shqToExqPortPerCycle : 1);
75+ exqIssueInflightCapPerPort_ = static_cast<int>(uarch.exqIssueInflightCapPerPort);
76+ computeInflightCap_ = static_cast<int>(uarch.computeInflightCap);
77+ shqReleaseDelay_ = static_cast<int>(uarch.shqReleaseDelay ? uarch.shqReleaseDelay : 1);
78+ iduVisiblePregDelay_ = static_cast<int>(uarch.iduVisiblePregDelay);
79+ iduVisibleShqDelay_ = static_cast<int>(uarch.iduVisibleShqDelay);
80+ visibleShqUsed_ = 0;
81+}
82+ 
83+int OoOCore::getFreePreg() const
84+{
85+ if (theoreticalLimitMode_)
86+ return 1000000000;
87+ if (enableCreditVisibilityDelay_)
88+ return std::max(0, visiblePregFree_);
89+ return static_cast<int>(freelist_.size());
90+}
91+ 
92+int OoOCore::getFreeShqQueue() const
93+{
94+ return theoreticalLimitMode_ ? 1000000000 : std::max(0, shqDepth_ - static_cast<int>(shq_.size()));
95+}
96+ 
97+int OoOCore::getFreeLsq() const
98+{
99+ return theoreticalLimitMode_ ? 1000000000 : std::max(0, lsqDepth_ - static_cast<int>(lsq_.size()));
100+}
101+ 
102+int OoOCore::getFreeShq() const
103+{
104+ if (theoreticalLimitMode_ || !enableShqCreditModel_)
105+ return 1000000000;
106+ if (enableCreditVisibilityDelay_)
107+ return std::max(0, shqDepth_ - visibleShqUsed_);
108+ return std::max(0, shqDepth_ - shqUsed_);
109+}
110+ 
111+std::unordered_map<std::string, int> OoOCore::updateIduVisibility(int64_t cycle)
112+{
113+ if (!enableCreditVisibilityDelay_)
114+ return std::unordered_map<std::string, int>{{"preg_free", 0}, {"shq_release", 0}};
115+ int pregDelta = iduMailboxPregReleaseDelta_;
116+ int shqDelta = iduMailboxShqReleaseDelta_;
117+ iduMailboxPregReleaseDelta_ = 0;
118+ iduMailboxShqReleaseDelta_ = 0;
119+ 
120+ auto pit = visiblePregFreeEvents_.find(cycle);
121+ if (pit != visiblePregFreeEvents_.end()) {
122+ visiblePregFree_ += pit->second;
123+ pregDelta += pit->second;
124+ visiblePregFreeEvents_.erase(pit);
125+ }
126+ auto sit = visibleShqReleaseEvents_.find(cycle);
127+ if (sit != visibleShqReleaseEvents_.end()) {
128+ visibleShqUsed_ = std::max(0, visibleShqUsed_ - sit->second);
129+ shqDelta += sit->second;
130+ visibleShqReleaseEvents_.erase(sit);
131+ }
132+ return std::unordered_map<std::string, int>{{"preg_free", pregDelta}, {"shq_release", shqDelta}};
133+}
134+ 
135+int64_t OoOCore::vfEndCycle() const { return lastDoneCycle_ + vfDrainCost_; }
136+ 
137+std::string OoOCore::classifyOpClass(const std::string& op, const std::string& form) const
138+{
139+ return isLoadOp(db_, op, form) ? "LOAD" : (isStoreOp(db_, op, form) ? "STORE" : "COMPUTE");
140+}
141+ 
142+bool OoOCore::isRegisterValue(const std::string& name) const { return valueStorage_.isRegister(name); }
143+ 
144+bool OoOCore::isUBValue(const std::string& name) const { return valueStorage_.isUB(name); }
145+ 
146+int64_t OoOCore::computeReadyTimeForSrc(
147+ const ProducerInfo& producerInfo, const std::string& consumerOp, const std::string& consumerForm) const
148+{
149+ const int64_t fwd = db_.forwardingCycles(producerInfo.op, producerInfo.form, consumerOp, consumerForm);
150+ if (isComputeOp(db_, consumerOp, consumerForm) && enableIsuQueueModel_)
151+ return producerInfo.startCycle + std::max<int64_t>(0, fwd - 1);
152+ return producerInfo.startCycle + fwd;
153+}
154+ 
155+int64_t OoOCore::computeLoadReadyCycle(const Uop& u) const
156+{
157+ int64_t t = std::max<int64_t>(vfStartupCost_, u.lsqReadyCycle);
158+ for (auto* pred : u.memDepUops) {
159+ if (pred && pred->doneCycle.has_value())
160+ t = std::max<int64_t>(t, pred->doneCycle.value());
161+ }
162+ if (memBarStrong_) {
163+ for (const auto& s : u.src) {
164+ if (!isIntermediateMemName(s))
165+ continue;
166+ if (u.topBlockId <= 0)
167+ continue;
168+ const auto it = blockReleaseCycle_.find(u.topBlockId - 1);
169+ if (it == blockReleaseCycle_.end())
170+ return 1000000000;
171+ t = std::max<int64_t>(t, it->second);
172+ }
173+ }
174+ return t;
175+}
176+ 
177+std::tuple<int64_t, std::optional<std::string>, std::optional<std::string>, std::optional<int64_t>>
178+OoOCore::computeStoreReadyCycle(const Uop& u) const
179+{
180+ for (const auto& ps : u.pregSrc) {
181+ if (!ps.has_value())
182+ continue;
183+ if (pregPending_.count(*ps) && pregProducer_.find(*ps) == pregProducer_.end())
184+ return {1000000000, std::nullopt, std::nullopt, std::nullopt};
185+ }
186+ 
187+ int64_t bestT = -1;
188+ std::optional<std::string> pop;
189+ std::optional<std::string> pform;
190+ std::optional<int64_t> pst;
191+ for (const auto& ps : u.pregSrc) {
192+ if (!ps.has_value())
193+ continue;
194+ auto it = pregProducer_.find(*ps);
195+ if (it == pregProducer_.end())
196+ continue;
197+ const auto& kind = it->second.kind;
198+ if (kind != "COMPUTE" && kind != "LOAD")
199+ continue;
200+ const int64_t cand = computeReadyTimeForSrc(it->second, u.op, u.form);
201+ if (cand > bestT) {
202+ bestT = cand;
203+ pop = it->second.op;
204+ pform = it->second.form;
205+ pst = it->second.startCycle;
206+ }
207+ }
208+ if (bestT < 0)
209+ return {1000000000, std::nullopt, std::nullopt, std::nullopt};
210+ bestT = std::max<int64_t>(bestT, u.lsqReadyCycle);
211+ return {bestT, pop, pform, pst};
212+}
213+ 
214+int64_t OoOCore::dataStoreCost(const std::string& producerOp, const std::string& producerForm) const
215+{
216+ const auto& cfg = db_.inst(producerOp, producerForm);
217+ return cfg.dataStoreCost > 0 ? cfg.dataStoreCost : 1;
218+}
219+ 
220+std::string OoOCore::getFuType(const std::string& op, const std::string& form) const
221+{
222+ const auto& cfg = db_.inst(op, form);
223+ std::string fu = cfg.exu.empty() ? "ALU" : cfg.exu;
224+ std::transform(
225+ fu.begin(), fu.end(), fu.begin(), [](unsigned char c) { return static_cast<char>(std::toupper(c)); });
226+ if (fu != "ALU" && fu != "SFU")
227+ fu = "ALU";
228+ return fu;
229+}
230+ 
231+std::vector<int> OoOCore::eligibleExuPorts(const std::string& op, const std::string& form) const
232+{
233+ std::string tag = db_.inst(op, form).dispatchExu;
234+ std::transform(
235+ tag.begin(), tag.end(), tag.begin(), [](unsigned char c) { return static_cast<char>(std::toupper(c)); });
236+ if (tag == "EXU0_ONLY")
237+ return issuePorts_ > 0 ? std::vector<int>{0} : std::vector<int>{};
238+ if (tag == "EXU01")
239+ return issuePorts_ >= 2 ? std::vector<int>{0, 1} : std::vector<int>{0};
240+ if (tag == "EXU012")
241+ return issuePorts_ >= 3 ? std::vector<int>{0, 1, 2} : std::vector<int>{0, 1};
242+ std::vector<int> out;
243+ for (int i = 0; i < issuePorts_; ++i)
244+ out.push_back(i);
245+ return out;
246+}
247+ 
248+int64_t OoOCore::getIi(
249+ const std::string* prevOp, const std::string* prevForm, const std::string& curOp, const std::string& curForm) const
250+{
251+ if (!prevOp || !prevForm || prevOp->empty())
252+ return 1;
253+ return db_.initiationInterval(*prevOp, *prevForm, curOp, curForm);
254+}
255+ 
256+void OoOCore::log(const std::string& event, const Uop& u)
257+{
258+ history_.push_back(HistoryRecord{
259+ cycle_, event, u.instId, u.op, u.state, u.readyCycle, u.startCycle, u.doneCycle, u.src, u.dst, u.pregSrc,
260+ u.pregDst, u.pregOld, u.producerOpForStore, u.producerStartForStore});
261+}
262+ 
263+void OoOCore::logStartSimple(const Uop& u)
264+{
265+ startLogs_.push_back(SimpleLogRecord{cycle_, u.instId, u.op, u.dst, u.src});
266+}
267+ 
268+void OoOCore::logDoneSimple(const Uop& u)
269+{
270+ doneLogs_.push_back(SimpleLogRecord{u.doneCycle.value_or(cycle_), u.instId, u.op, u.dst, u.src});
271+}
272+ 
273+void OoOCore::dumpHistory(const std::string& path) const
274+{
275+ std::ofstream os(path);
276+ os << "[\n";
277+ for (size_t i = 0; i < history_.size(); ++i) {
278+ const auto& h = history_[i];
279+ os << " {"
280+ << "\"cy\":" << h.cy << ","
281+ << "\"event\":\"" << jsonEscape(h.event) << "\","
282+ << "\"id\":" << h.id << ","
283+ << "\"op\":\"" << jsonEscape(h.op) << "\","
284+ << "\"state\":\"" << jsonEscape(h.state) << "\","
285+ << "\"ready\":" << h.ready << ","
286+ << "\"start\":" << (h.start ? std::to_string(*h.start) : "null") << ","
287+ << "\"done\":" << (h.done ? std::to_string(*h.done) : "null") << ","
288+ << "\"src\":" << joinJsonArray(h.src) << ","
289+ << "\"dst\":" << joinJsonArray(h.dst) << ","
290+ << "\"preg_src\":" << joinJsonArray(h.pregSrc) << ","
291+ << "\"preg_dst\":" << joinJsonArray(h.pregDst) << ","
292+ << "\"preg_old\":" << joinJsonArray(h.pregOld) << ","
293+ << "\"producer_op_for_store\":"
294+ << (h.producerOpForStore ? "\"" + jsonEscape(*h.producerOpForStore) + "\"" : "null") << ","
295+ << "\"producer_start_for_store\":"
296+ << (h.producerStartForStore ? std::to_string(*h.producerStartForStore) : "null") << "}";
297+ if (i + 1 < history_.size())
298+ os << ",";
299+ os << "\n";
300+ }
301+ os << "]\n";
302+}
303+ 
304+void OoOCore::dumpSimpleLogs(const std::string& startPath, const std::string& donePath) const
305+{
306+ std::ofstream s(startPath);
307+ for (const auto& r : startLogs_) {
308+ s << "{\"cy\":" << r.cy << ",\"inst_id\":" << r.instId << ",\"op\":\"" << jsonEscape(r.op)
309+ << "\",\"dst\":" << joinJsonArray(r.dst) << ",\"src\":" << joinJsonArray(r.src) << "}\n";
310+ }
311+ std::ofstream d(donePath);
312+ for (const auto& r : doneLogs_) {
313+ d << "{\"cy\":" << r.cy << ",\"inst_id\":" << r.instId << ",\"op\":\"" << jsonEscape(r.op)
314+ << "\",\"dst\":" << joinJsonArray(r.dst) << ",\"src\":" << joinJsonArray(r.src) << "}\n";
315+ }
316+}
317+ 
318+Uop* OoOCore::findRobUop(int64_t instId)
319+{
320+ for (auto& u : rob_) {
321+ if (u.instId == instId)
322+ return &u;
323+ }
324+ return nullptr;
325+}
326+ 
327+bool OoOCore::isCurrentMapping(const std::string& preg) const
328+{
329+ for (const auto& [_, cur] : rat_) {
330+ if (cur == preg)
331+ return true;
332+ }
333+ return false;
334+}
335+ 
336+void OoOCore::scheduleSrcReleaseFromStart(const Uop& u)
337+{
338+ if (!u.startCycle.has_value())
339+ return;
340+ if (srcReleaseScheduledInstIds_.count(u.instId))
341+ return;
342+ const int64_t releaseCycle = *u.startCycle + consumerReleaseStartOffset_;
343+ auto& bucket = srcReleaseEvents_[releaseCycle];
344+ for (size_t i = 0; i < u.pregSrc.size(); ++i) {
345+ const auto& s = u.pregSrc[i];
346+ if (!s.has_value())
347+ continue;
348+ const int64_t gen =
349+ (i < u.pregSrcGen.size() && u.pregSrcGen[i].has_value()) ? *u.pregSrcGen[i] : pregGeneration_[*s];
350+ bucket.push_back(SrcReleaseEvent{u.instId, *s, gen});
351+ srcReleaseExpected_[u.instId] += 1;
352+ }
353+ srcReleaseSeen_[u.instId] = 0;
354+ srcReleaseScheduledInstIds_.insert(u.instId);
355+}
356+ 
357+void OoOCore::runSrcReleaseEvents(int64_t cycle)
358+{
359+ auto it = srcReleaseEvents_.find(cycle);
360+ if (it == srcReleaseEvents_.end())
361+ return;
362+ for (const auto& ev : it->second) {
363+ ++srcReleaseSeen_[ev.instId];
364+ const auto genIt = pregGeneration_.find(ev.preg);
365+ if (genIt == pregGeneration_.end() || genIt->second != ev.gen)
366+ continue;
367+ auto cntIt = pregConsumerCount_.find(ev.preg);
368+ if (cntIt != pregConsumerCount_.end() && cntIt->second > 0) {
369+ --cntIt->second;
370+ if (cntIt->second == 0)
371+ pregReleaseEligibleCycle_[ev.preg] = cycle;
372+ }
373+ }
374+ srcReleaseEvents_.erase(it);
375+}
376+ 
377+bool OoOCore::tryFreePreg(const std::string& preg, int64_t cycle)
378+{
379+ if (preg.empty() || isCurrentMapping(preg))
380+ return false;
381+ auto cntIt = pregConsumerCount_.find(preg);
382+ if (cntIt != pregConsumerCount_.end() && cntIt->second > 0)
383+ return false;
384+ auto eligIt = pregReleaseEligibleCycle_.find(preg);
385+ if (eligIt != pregReleaseEligibleCycle_.end() && cycle < eligIt->second)
386+ return false;
387+ if (std::find(freelist_.begin(), freelist_.end(), preg) != freelist_.end())
388+ return false;
389+ if (pregPending_.count(preg))
390+ return false;
391+ pregProducer_.erase(preg);
392+ pregPending_.erase(preg);
393+ pregConsumerCount_.erase(preg);
394+ pregReleaseEligibleCycle_.erase(preg);
395+ freelist_.push_back(preg);
396+ if (enableCreditVisibilityDelay_) {
397+ if (iduVisiblePregDelay_ <= 0) {
398+ ++visiblePregFree_;
399+ ++iduMailboxPregReleaseDelta_;
400+ } else {
401+ visiblePregFreeEvents_[cycle + iduVisiblePregDelay_] += 1;
402+ }
403+ }
404+ return true;
405+}
406+ 
407+void OoOCore::tryFreeEligiblePregs(int64_t cycle)
408+{
409+ std::vector<std::string> elig;
410+ elig.reserve(pregReleaseEligibleCycle_.size());
411+ for (const auto& [preg, _] : pregReleaseEligibleCycle_)
412+ elig.push_back(preg);
413+ for (const auto& preg : elig)
414+ (void)tryFreePreg(preg, cycle);
415+}
416+ 
417+int OoOCore::exqOccupancy(int port) const
418+{
419+ if (port < 0 || port >= static_cast<int>(exqInflight_.size()))
420+ return 0;
421+ return exqInflight_[static_cast<size_t>(port)] * (exqCapacityCountsInflight_ ? 1 : 0);
422+}
423+ 
424+int OoOCore::totalComputeInflight() const
425+{
426+ int total = 0;
427+ for (int x : exqInflight_)
428+ total += x;
429+ return total;
430+}
431+ 
432+int64_t OoOCore::predictExqIssueCycle(
433+ int port, const std::string& fuType, const std::string& op, const std::string& form, int64_t recvCycle) const
434+{
435+ int64_t pred = recvCycle;
436+ const std::string* prevOp = nullptr;
437+ const std::string* prevForm = nullptr;
438+ int64_t prevIssue = -1000000000;
439+ if (enableCrossFuIi_) {
440+ prevOp = &lastOpExu_[static_cast<size_t>(port)];
441+ prevForm = &lastFormExu_[static_cast<size_t>(port)];
442+ prevIssue = lastIssueCycleExu_[static_cast<size_t>(port)];
443+ } else if (fuType == "SFU") {
444+ prevOp = &lastOpSFU_[static_cast<size_t>(port)];
445+ prevForm = &lastFormSFU_[static_cast<size_t>(port)];
446+ prevIssue = lastIssueCycleSFU_[static_cast<size_t>(port)];
447+ } else {
448+ prevOp = &lastOpALU_[static_cast<size_t>(port)];
449+ prevForm = &lastFormALU_[static_cast<size_t>(port)];
450+ prevIssue = lastIssueCycleALU_[static_cast<size_t>(port)];
451+ }
452+ pred = std::max<int64_t>(pred, prevIssue + getIi(prevOp, prevForm, op, form));
453+ return pred;
454+}
455+ 
456+void OoOCore::scheduleShqRelease(int64_t cycle, int count)
457+{
458+ if (!enableShqCreditModel_ || count <= 0)
459+ return;
460+ shqReleaseEvents_[cycle + shqReleaseDelay_] += count;
461+}
462+ 
463+void OoOCore::runShqReleaseEvents(int64_t cycle)
464+{
465+ if (!enableShqCreditModel_)
466+ return;
467+ auto it = shqReleaseEvents_.find(cycle);
468+ if (it == shqReleaseEvents_.end())
469+ return;
470+ const int released = it->second;
471+ shqReleaseEvents_.erase(it);
472+ shqUsed_ = std::max(0, shqUsed_ - released);
473+ if (!enableCreditVisibilityDelay_)
474+ return;
475+ if (iduVisibleShqDelay_ <= 0) {
476+ visibleShqUsed_ = std::max(0, visibleShqUsed_ - released);
477+ iduMailboxShqReleaseDelta_ += released;
478+ } else {
479+ visibleShqReleaseEvents_[cycle + iduVisibleShqDelay_] += released;
480+ }
481+}
482+ 
483+OoOCoreMainline::OoOCoreMainline(
484+ const UarchConfig& uarch, const ParamDB& db, std::string dtype,
485+ const std::unordered_map<std::string, ValueInfo>& values)
486+ : OoOCore(uarch, db, std::move(dtype), values)
487+{
488+ exqInflightPerPort_.assign(issuePorts_, 0);
489+ exqWait_.resize(static_cast<size_t>(issuePorts_));
490+ for (auto& port : exqWait_) {
491+ port["ALU"] = std::deque<Uop>{};
492+ port["SFU"] = std::deque<Uop>{};
493+ }
494+ consumerReleaseStartOffset_ = static_cast<int>(uarch.consumerReleaseStartOffset);
495+}
496+ 
497+void OoOCoreMainline::accept(const DynamicInst& inst)
498+{
499+ Uop u = makeAcceptedUop(inst);
500+ capturePregSources(u);
501+ addPregConsumers(u);
502+ const int allocCount = allocatePregDsts(u);
503+ if (enableCreditVisibilityDelay_ && allocCount > 0)
504+ visiblePregFree_ = std::max(0, visiblePregFree_ - allocCount);
505+ trackSharedShqCredit(u);
506+ enqueueAcceptedUop(u);
507+ rob_.push_back(u);
508+ if (isStoreOp(db_, u.op, u.form))
509+ blockOutstandingStores_[u.topBlockId] += 1;
510+ releaseOldMappings(u);
511+}
512+ 
513+Uop OoOCoreMainline::makeAcceptedUop(const DynamicInst& inst) const
514+{
515+ Uop u;
516+ u.instId = inst.instId;
517+ u.op = inst.op;
518+ u.form = inst.form.empty() ? dtype_ : inst.form;
519+ u.src = inst.src;
520+ u.dst = inst.dst;
521+ u.topBlockId = inst.topBlockId;
522+ u.iterStack = inst.iterStack;
523+ u.isLastInTopBlock = inst.isLastInTopBlock;
524+ return u;
525+}
526+ 
527+void OoOCoreMainline::capturePregSources(Uop& u) const
528+{
529+ for (const auto& s : u.src) {
530+ if (!isRegisterValue(s)) {
531+ u.pregSrc.push_back(std::nullopt);
532+ u.pregSrcGen.push_back(std::nullopt);
533+ continue;
534+ }
535+ auto it = rat_.find(s);
536+ if (it == rat_.end()) {
537+ u.pregSrc.push_back(std::nullopt);
538+ u.pregSrcGen.push_back(std::nullopt);
539+ } else {
540+ u.pregSrc.push_back(it->second);
541+ auto genIt = pregGeneration_.find(it->second);
542+ u.pregSrcGen.push_back(
543+ genIt == pregGeneration_.end() ? std::optional<int64_t>{0} : std::optional<int64_t>{genIt->second});
544+ }
545+ }
546+}
547+ 
548+void OoOCoreMainline::addPregConsumers(const Uop& u)
549+{
550+ for (const auto& preg : u.pregSrc) {
551+ if (preg) {
552+ pregConsumerCount_[*preg] += 1;
553+ pregReleaseEligibleCycle_.erase(*preg);
554+ }
555+ }
556+}
557+ 
558+int OoOCoreMainline::allocatePregDsts(Uop& u)
559+{
560+ int allocCount = 0;
561+ for (const auto& d : u.dst) {
562+ if (!isRegisterValue(d)) {
563+ u.pregDst.push_back(std::string{});
564+ continue;
565+ }
566+ std::string newPreg;
567+ if (theoreticalLimitMode_ || freelist_.empty()) {
568+ newPreg = "p" + std::to_string(nextDynamicPregId_++);
569+ } else {
570+ newPreg = freelist_.front();
571+ freelist_.pop_front();
572+ }
573+ std::string oldPreg;
574+ auto rit = rat_.find(d);
575+ if (rit != rat_.end())
576+ oldPreg = rit->second;
577+ rat_[d] = newPreg;
578+ u.pregDst.push_back(newPreg);
579+ u.pregOld.push_back(oldPreg.empty() ? std::optional<std::string>{} : std::optional<std::string>{oldPreg});
580+ pregGeneration_[newPreg] += 1;
581+ pregConsumerCount_[newPreg] = 0;
582+ pregPending_.insert(newPreg);
583+ pregReleaseEligibleCycle_.erase(newPreg);
584+ ++allocCount;
585+ }
586+ return allocCount;
587+}
588+ 
589+void OoOCoreMainline::trackSharedShqCredit(Uop& u)
590+{
591+ if (enableShqCreditModel_ && usesSharedShqCredit(db_, u.op, u.form)) {
592+ ++shqUsed_;
593+ if (enableCreditVisibilityDelay_)
594+ ++visibleShqUsed_;
595+ u.isShqTracked = true;
596+ }
597+}
598+ 
599+void OoOCoreMainline::enqueueAcceptedUop(Uop& u)
600+{
601+ if (usesLsq(db_, u.op, u.form)) {
602+ u.lsqReadyCycle = static_cast<int64_t>(cycle_) + oooToLsqDelay_;
603+ lsq_.push_back(u);
604+ return;
605+ }
606+ u.shqReadyCycle = static_cast<int64_t>(cycle_) + oooToShqDelay_;
607+ shq_.push_back(u);
608+}
609+ 
610+void OoOCoreMainline::releaseOldMappings(const Uop& u)
611+{
612+ for (const auto& oldPreg : u.pregOld) {
613+ if (oldPreg)
614+ (void)tryFreePreg(*oldPreg, cycle_);
615+ }
616+}
617+ 
618+void OoOCoreMainline::freeOldPregs(const Uop& u)
619+{
620+ for (const auto& oldPreg : u.pregOld) {
621+ if (!oldPreg.has_value())
622+ continue;
623+ (void)tryFreePreg(*oldPreg, cycle_);
624+ }
625+}
626+ 
627+void OoOCoreMainline::completeRunningUops(int64_t cycle)
628+{
629+ for (auto& u : rob_) {
630+ if (u.state != "running" || !u.doneCycle.has_value() || cycle < *u.doneCycle) {
631+ continue;
632+ }
633+ u.state = "done";
634+ if (u.exuPort >= 0 && u.exuPort < static_cast<int>(exqInflight_.size())) {
635+ auto& inflight = exqInflight_[static_cast<size_t>(u.exuPort)];
636+ inflight = std::max(0, inflight - 1);
637+ }
638+ if (u.isLastInTopBlock)
639+ blockLastInstDone_[u.topBlockId] = true;
640+ if (isStoreOp(db_, u.op, u.form)) {
641+ auto it = blockOutstandingStores_.find(u.topBlockId);
642+ if (it != blockOutstandingStores_.end())
643+ it->second = std::max(0, it->second - 1);
644+ }
645+ if (blockLastInstDone_[u.topBlockId] && blockOutstandingStores_[u.topBlockId] == 0) {
646+ auto prev = blockReleaseCycle_.find(u.topBlockId);
647+ if (prev == blockReleaseCycle_.end())
648+ blockReleaseCycle_[u.topBlockId] = *u.doneCycle;
649+ else
650+ prev->second = std::max<int64_t>(prev->second, *u.doneCycle);
651+ }
652+ log("done", u);
653+ logDoneSimple(u);
654+ lastDoneCycle_ = std::max(lastDoneCycle_, *u.doneCycle);
655+ for (const auto& pd : u.pregDst) {
656+ if (!pd.empty())
657+ (void)tryFreePreg(pd, cycle);
658+ }
659+ }
660+}
661+ 
662+void OoOCoreMainline::retireDoneUops()
663+{
664+ while (!rob_.empty() && rob_.front().state == "done") {
665+ Uop u = rob_.front();
666+ rob_.pop_front();
667+ freeOldPregs(u);
668+ log("retire", u);
669+ }
670+}
671+ 
672+int64_t OoOCoreMainline::computeShqReadyCycle(const Uop& u) const
673+{
674+ int64_t ready = std::max<int64_t>(vfStartupCost_, u.shqReadyCycle);
675+ for (const auto& preg : u.pregSrc) {
676+ if (!preg.has_value())
677+ continue;
678+ auto it = pregProducer_.find(*preg);
679+ if (it == pregProducer_.end()) {
680+ if (pregPending_.count(*preg))
681+ ready = std::max<int64_t>(ready, 1000000000);
682+ continue;
683+ }
684+ ready = std::max<int64_t>(ready, computeReadyTimeForSrc(it->second, u.op, u.form));
685+ }
686+ return ready;
687+}
688+ 
689+void OoOCoreMainline::updateLsqReadiness(int64_t cycle)
690+{
691+ for (auto& u : lsq_) {
692+ if (u.state == "running" || u.state == "done")
693+ continue;
694+ if (isLoadOp(db_, u.op, u.form))
695+ u.readyCycle = computeLoadReadyCycle(u);
696+ else
697+ u.readyCycle = std::get<0>(computeStoreReadyCycle(u));
698+ u.state = (cycle >= u.readyCycle) ? "ready" : "blocked";
699+ }
700+}
701+ 
702+void OoOCoreMainline::updateShqReadiness(int64_t cycle)
703+{
704+ for (auto& u : shq_) {
705+ if (u.state == "running" || u.state == "done")
706+ continue;
707+ u.readyCycle = computeShqReadyCycle(u);
708+ u.state = (cycle >= u.readyCycle) ? "ready" : "blocked";
709+ }
710+}
711+ 
712+void OoOCoreMainline::mirrorStartedUopToRob(const Uop& u)
713+{
714+ if (auto* robU = findRobUop(u.instId)) {
715+ robU->startCycle = u.startCycle;
716+ robU->doneCycle = u.doneCycle;
717+ robU->state = u.state;
718+ robU->exuPort = u.exuPort;
719+ }
720+}
721+ 
722+void OoOCoreMainline::markProducerReady(const Uop& u, const std::string& kind)
723+{
724+ for (const auto& pd : u.pregDst) {
725+ if (pd.empty())
726+ continue;
727+ pregProducer_[pd] = ProducerInfo{u.op, u.form, *u.startCycle, kind};
728+ pregPending_.erase(pd);
729+ }
730+}
731+ 
732+bool OoOCoreMainline::hasSameCycleSrcHazard(const Uop& u, const std::unordered_set<std::string>& issuedSrcs) const
733+{
734+ if (!enforceSameCycleSrcHazard_ || theoreticalLimitMode_)
735+ return false;
736+ for (const auto& ps : u.pregSrc) {
737+ if (ps && issuedSrcs.count(*ps))
738+ return true;
739+ }
740+ return false;
741+}
742+ 
743+void OoOCoreMainline::rememberIssuedSrcs(const Uop& u, std::unordered_set<std::string>& issuedSrcs) const
744+{
745+ for (const auto& ps : u.pregSrc) {
746+ if (ps)
747+ issuedSrcs.insert(*ps);
748+ }
749+}
750+ 
751+void OoOCoreMainline::issueReadyLoads(int64_t cycle)
752+{
753+ int ld = 0;
754+ for (auto it = lsq_.begin(); it != lsq_.end();) {
755+ auto& u = *it;
756+ if (u.state != "ready" || !isLoadOp(db_, u.op, u.form)) {
757+ ++it;
758+ continue;
759+ }
760+ if (ld >= loadPorts_)
761+ break;
762+ u.startCycle = cycle;
763+ u.doneCycle = cycle + loadDoneLatency_;
764+ u.state = "running";
765+ scheduleSrcReleaseFromStart(u);
766+ mirrorStartedUopToRob(u);
767+ log("start", u);
768+ logStartSimple(u);
769+ ++ld;
770+ markProducerReady(u, "LOAD");
771+ it = lsq_.erase(it);
772+ }
773+}
774+ 
775+int OoOCoreMainline::chooseDirectIssuePort(const Uop& u, const std::vector<bool>& exuUsedThisCycle) const
776+{
777+ const std::string fuType = getFuType(u.op, u.form);
778+ for (int port : eligibleExuPorts(u.op, u.form)) {
779+ if (port < 0 || port >= issuePorts_ || exuUsedThisCycle[static_cast<size_t>(port)])
780+ continue;
781+ const std::string* prevOp = enableCrossFuIi_ ? &lastOpExu_[static_cast<size_t>(port)] :
782+ (fuType == "SFU" ? &lastOpSFU_[static_cast<size_t>(port)] :
783+ &lastOpALU_[static_cast<size_t>(port)]);
784+ const std::string* prevForm = enableCrossFuIi_ ? &lastFormExu_[static_cast<size_t>(port)] :
785+ (fuType == "SFU" ? &lastFormSFU_[static_cast<size_t>(port)] :
786+ &lastFormALU_[static_cast<size_t>(port)]);
787+ const int64_t prevIssue = enableCrossFuIi_ ? lastIssueCycleExu_[static_cast<size_t>(port)] :
788+ (fuType == "SFU" ? lastIssueCycleSFU_[static_cast<size_t>(port)] :
789+ lastIssueCycleALU_[static_cast<size_t>(port)]);
790+ if (cycle_ >= prevIssue + getIi(prevOp, prevForm, u.op, u.form))
791+ return port;
792+ }
793+ return -1;
794+}
795+ 
796+void OoOCoreMainline::recordComputeIssue(Uop& u, int64_t cycle, int port, const std::string& fuType)
797+{
798+ u.startCycle = cycle;
799+ u.doneCycle = cycle + std::max<int64_t>(1, db_.inst(u.op, u.form).latency);
800+ u.state = "running";
801+ u.exuPort = port;
802+ scheduleSrcReleaseFromStart(u);
803+ mirrorStartedUopToRob(u);
804+ log("start", u);
805+ logStartSimple(u);
806+ if (enableCrossFuIi_) {
807+ lastIssueCycleExu_[static_cast<size_t>(port)] = cycle;
808+ lastOpExu_[static_cast<size_t>(port)] = u.op;
809+ lastFormExu_[static_cast<size_t>(port)] = u.form;
810+ } else if (fuType == "SFU") {
811+ lastIssueCycleSFU_[static_cast<size_t>(port)] = cycle;
812+ lastOpSFU_[static_cast<size_t>(port)] = u.op;
813+ lastFormSFU_[static_cast<size_t>(port)] = u.form;
814+ } else {
815+ lastIssueCycleALU_[static_cast<size_t>(port)] = cycle;
816+ lastOpALU_[static_cast<size_t>(port)] = u.op;
817+ lastFormALU_[static_cast<size_t>(port)] = u.form;
818+ }
819+ exqInflight_[static_cast<size_t>(port)] += 1;
820+ markProducerReady(u, "COMPUTE");
821+}
822+ 
823+void OoOCoreMainline::issueReadyComputeDirect(
824+ int64_t cycle, std::vector<bool>& exuUsedThisCycle, std::unordered_set<std::string>& issuedSrcsThisCycle)
825+{
826+ int issued = 0;
827+ for (auto it = shq_.begin(); it != shq_.end();) {
828+ auto& u = *it;
829+ if (u.state != "ready" || isLoadOp(db_, u.op, u.form) || isStoreOp(db_, u.op, u.form)) {
830+ ++it;
831+ continue;
832+ }
833+ if (issued >= issuePorts_)
834+ break;
835+ if (hasSameCycleSrcHazard(u, issuedSrcsThisCycle)) {
836+ ++it;
837+ continue;
838+ }
839+ const std::string fuType = getFuType(u.op, u.form);
840+ const int chosenPort = chooseDirectIssuePort(u, exuUsedThisCycle);
841+ if (chosenPort < 0) {
842+ ++it;
843+ continue;
844+ }
845+ recordComputeIssue(u, cycle, chosenPort, fuType);
846+ ++issued;
847+ exuUsedThisCycle[static_cast<size_t>(chosenPort)] = true;
848+ rememberIssuedSrcs(u, issuedSrcsThisCycle);
849+ it = shq_.erase(it);
850+ }
851+}
852+ 
853+void OoOCoreMainline::enqueueReadyComputeToExq(int64_t cycle, std::unordered_set<std::string>& issuedSrcsThisCycle)
854+{
855+ std::vector<int> shqToExqCnt(static_cast<size_t>(issuePorts_), 0);
856+ int exCount = 0;
857+ for (auto it = shq_.begin(); it != shq_.end();) {
858+ auto& u = *it;
859+ if (u.state != "ready") {
860+ ++it;
861+ continue;
862+ }
863+ if (exCount >= issuePorts_)
864+ break;
865+ if (hasSameCycleSrcHazard(u, issuedSrcsThisCycle)) {
866+ ++it;
867+ continue;
868+ }
869+ const std::string fuType = getFuType(u.op, u.form);
870+ const ExqChoice choice = chooseExqEnqueuePort(u, fuType, shqToExqCnt, cycle);
871+ if (choice.port < 0) {
872+ ++it;
873+ continue;
874+ }
875+ moveUopToExqWait(u, cycle, choice, fuType);
876+ shqToExqCnt[static_cast<size_t>(choice.port)] += 1;
877+ ++exCount;
878+ rememberIssuedSrcs(u, issuedSrcsThisCycle);
879+ it = shq_.erase(it);
880+ }
881+}
882+ 
883+OoOCoreMainline::ExqChoice OoOCoreMainline::chooseExqEnqueuePort(
884+ const Uop& u, const std::string& fuType, const std::vector<int>& shqToExqCnt, int64_t cycle) const
885+{
886+ ExqChoice choice;
887+ for (int port : eligibleExuPorts(u.op, u.form)) {
888+ if (port < 0 || port >= issuePorts_)
889+ continue;
890+ if (shqToExqCnt[static_cast<size_t>(port)] >= shqToExqPortPerCycle_)
891+ continue;
892+ const auto& q = exqWait_[static_cast<size_t>(port)];
893+ int occ = static_cast<int>(q.at("ALU").size() + q.at("SFU").size());
894+ if (exqCapacityCountsInflight_)
895+ occ += exqInflight_[static_cast<size_t>(port)];
896+ if (occ >= exqDepth_)
897+ continue;
898+ const int64_t recv = cycle + exqRecvDelay_;
899+ int64_t pred = recv;
900+ const auto& fq = q.at(fuType);
901+ if (!fq.empty()) {
902+ const Uop& prev = fq.back();
903+ pred = std::max<int64_t>(pred, prev.exqPredIssue + getIi(&prev.op, &prev.form, u.op, u.form));
904+ } else {
905+ pred = std::max<int64_t>(pred, predictExqIssueCycle(port, fuType, u.op, u.form, recv));
906+ }
907+ const auto key = std::make_tuple(pred, occ, port);
908+ const auto best = std::make_tuple(choice.pred, choice.occ, choice.port);
909+ if (choice.port < 0 || key < best)
910+ choice = ExqChoice{port, pred, occ};
911+ }
912+ return choice;
913+}
914+ 
915+void OoOCoreMainline::moveUopToExqWait(Uop& u, int64_t cycle, const ExqChoice& choice, const std::string& fuType)
916+{
917+ u.exuPort = choice.port;
918+ u.exqRecvCycle = cycle + exqRecvDelay_;
919+ u.exqPredIssue = choice.pred;
920+ u.state = "exq_wait";
921+ if (usesSharedShqCredit(db_, u.op, u.form)) {
922+ scheduleShqRelease(cycle, 1);
923+ u.isShqTracked = false;
924+ if (auto* robU = findRobUop(u.instId))
925+ robU->isShqTracked = false;
926+ }
927+ exqWait_[static_cast<size_t>(choice.port)][fuType].push_back(u);
928+}
929+ 
930+Uop* OoOCoreMainline::selectExqIssueCandidate(int port, int64_t cycle, std::string& fuType)
931+{
932+ auto& q = exqWait_[static_cast<size_t>(port)];
933+ Uop* bestU = nullptr;
934+ std::tuple<int64_t, int64_t, int64_t> bestKey{0, 0, 0};
935+ for (const std::string& candidateFu : {std::string("ALU"), std::string("SFU")}) {
936+ auto& fq = q[candidateFu];
937+ if (fq.empty())
938+ continue;
939+ Uop& cand = fq.front();
940+ if (exqIssueInflightCapPerPort_ > 0 && exqInflight_[static_cast<size_t>(port)] >= exqIssueInflightCapPerPort_)
941+ continue;
942+ if (cand.exqRecvCycle > cycle)
943+ continue;
944+ const int64_t ready = computeShqReadyCycle(cand);
945+ if (ready > cycle)
946+ continue;
947+ const int64_t ii =
948+ getIi(&lastOpExu_[static_cast<size_t>(port)], &lastFormExu_[static_cast<size_t>(port)], cand.op, cand.form);
949+ if (cycle < lastIssueCycleExu_[static_cast<size_t>(port)] + ii)
950+ continue;
951+ auto key = std::make_tuple(ready, cand.exqRecvCycle, cand.instId);
952+ if (!bestU || key < bestKey) {
953+ fuType = candidateFu;
954+ bestU = &cand;
955+ bestKey = key;
956+ }
957+ }
958+ return bestU;
959+}
960+ 
961+void OoOCoreMainline::issueExqWaitQueues(int64_t cycle, std::vector<bool>& exuUsedThisCycle)
962+{
963+ for (int port = 0; port < issuePorts_; ++port) {
964+ if (exuUsedThisCycle[static_cast<size_t>(port)])
965+ continue;
966+ std::string bestFu;
967+ Uop* bestU = selectExqIssueCandidate(port, cycle, bestFu);
968+ if (!bestU)
969+ continue;
970+ Uop u = *bestU;
971+ exqWait_[static_cast<size_t>(port)][bestFu].pop_front();
972+ recordComputeIssue(u, cycle, port, bestFu);
973+ lastIssueCycleExu_[static_cast<size_t>(port)] = cycle;
974+ lastOpExu_[static_cast<size_t>(port)] = u.op;
975+ lastFormExu_[static_cast<size_t>(port)] = u.form;
976+ exuUsedThisCycle[static_cast<size_t>(port)] = true;
977+ }
978+}
979+ 
980+void OoOCoreMainline::issueReadyComputeViaIsu(
981+ int64_t cycle, std::vector<bool>& exuUsedThisCycle, std::unordered_set<std::string>& issuedSrcsThisCycle)
982+{
983+ enqueueReadyComputeToExq(cycle, issuedSrcsThisCycle);
984+ issueExqWaitQueues(cycle, exuUsedThisCycle);
985+}
986+ 
987+void OoOCoreMainline::issueReadyStores(int64_t cycle)
988+{
989+ int st = 0;
990+ for (auto it = lsq_.begin(); it != lsq_.end();) {
991+ auto& u = *it;
992+ if (u.state != "ready" || !isStoreOp(db_, u.op, u.form)) {
993+ ++it;
994+ continue;
995+ }
996+ if (st >= storePorts_)
997+ break;
998+ auto ready = computeStoreReadyCycle(u);
999+ if (cycle < std::get<0>(ready)) {
1000+ ++it;
1001+ continue;
1002+ }
1003+ u.producerOpForStore = std::get<1>(ready);
1004+ u.producerFormForStore = std::get<2>(ready);
1005+ u.producerStartForStore = std::get<3>(ready);
1006+ if (!u.producerOpForStore.has_value()) {
1007+ ++it;
1008+ continue;
1009+ }
1010+ u.startCycle = cycle;
1011+ u.doneCycle = cycle + dataStoreCost(*u.producerOpForStore, u.producerFormForStore.value_or(u.form));
1012+ u.state = "running";
1013+ scheduleSrcReleaseFromStart(u);
1014+ if (usesSharedShqCredit(db_, u.op, u.form)) {
1015+ scheduleShqRelease(cycle, 1);
1016+ u.isShqTracked = false;
1017+ }
1018+ if (auto* robU = findRobUop(u.instId)) {
1019+ robU->producerOpForStore = u.producerOpForStore;
1020+ robU->producerStartForStore = u.producerStartForStore;
1021+ robU->startCycle = u.startCycle;
1022+ robU->doneCycle = u.doneCycle;
1023+ robU->state = u.state;
1024+ robU->isShqTracked = false;
1025+ }
1026+ log("start", u);
1027+ logStartSimple(u);
1028+ ++st;
1029+ it = lsq_.erase(it);
1030+ }
1031+}
1032+ 
1033+void OoOCoreMainline::step()
1034+{
1035+ const int64_t c = cycle_;
1036+ 
1037+ runShqReleaseEvents(c);
1038+ runSrcReleaseEvents(c);
1039+ completeRunningUops(c);
1040+ retireDoneUops();
1041+ tryFreeEligiblePregs(c);
1042+ 
1043+ updateLsqReadiness(c);
1044+ updateShqReadiness(c);
1045+ issueReadyLoads(c);
1046+ updateShqReadiness(c);
1047+ 
1048+ std::vector<bool> exuUsedThisCycle(static_cast<size_t>(issuePorts_), false);
1049+ std::unordered_set<std::string> issuedSrcsThisCycle;
1050+ if (!enableIsuQueueModel_) {
1051+ issueReadyComputeDirect(c, exuUsedThisCycle, issuedSrcsThisCycle);
1052+ } else {
1053+ issueReadyComputeViaIsu(c, exuUsedThisCycle, issuedSrcsThisCycle);
1054+ }
1055+ issueReadyStores(c);
1056+ 
1057+ ++cycle_;
1058+}
1059+ 
1060+} // namespace vfsim