已合并
[forceinline] add Kunpeng forceinline optimization #66
huang-xiaoquan创建于 5月18日
[forceinline] add Kunpeng forceinline optimization #66
已合并
共 3 个文件变更+275-4
| @@ -75,6 +75,8 @@ type DebugFlags struct { | |||
| 75 | WrapGlobalMapDbg int `help:"debug trace output for global map init wrapping"` | 75 | WrapGlobalMapDbg int `help:"debug trace output for global map init wrapping"` |
| 76 | WrapGlobalMapCtl int `help:"global map init wrap control (0 => default, 1 => off, 2 => stress mode, no size cutoff)"` | 76 | WrapGlobalMapCtl int `help:"global map init wrap control (0 => default, 1 => off, 2 => stress mode, no size cutoff)"` |
| 77 | ZeroCopy int `help:"enable zero-copy string->[]byte conversions" concurrent:"ok"` | 77 | ZeroCopy int `help:"enable zero-copy string->[]byte conversions" concurrent:"ok"` |
| 78 | + ForceInline int `help:"enable force inline"` | ||
| 79 | + ForceInlineLog int `help:"print force inline log"` | ||
| 78 | 80 | ||
| 79 | ConcurrentOk bool // true if only concurrentOk flags seen | 81 | ConcurrentOk bool // true if only concurrentOk flags seen |
| 80 | } | 82 | } |
| @@ -43,6 +43,8 @@ import ( | |||
| 43 | "cmd/internal/obj" | 43 | "cmd/internal/obj" |
| 44 | "cmd/internal/pgo" | 44 | "cmd/internal/pgo" |
| 45 | "cmd/internal/src" | 45 | "cmd/internal/src" |
| 46 | + "math" | ||
| 47 | + "sync" | ||
| 46 | ) | 48 | ) |
| 47 | 49 | ||
| 48 | // Inlining budget parameters, gathered in one place | 50 | // Inlining budget parameters, gathered in one place |
| @@ -58,6 +60,9 @@ const ( | |||
| 58 | inlineBigFunctionNodes = 5000 // Functions with this many nodes are considered "big". | 60 | inlineBigFunctionNodes = 5000 // Functions with this many nodes are considered "big". |
| 59 | inlineBigFunctionMaxCost = 20 // Max cost of inlinee when inlining into a "big" function. | 61 | inlineBigFunctionMaxCost = 20 // Max cost of inlinee when inlining into a "big" function. |
| 60 | inlineClosureCalledOnceCost = 10 * inlineMaxBudget // if a closure is just called once, inline it. | 62 | inlineClosureCalledOnceCost = 10 * inlineMaxBudget // if a closure is just called once, inline it. |
| 63 | + | ||
| 64 | + // The hairyVisitor.budget may be increased in some place, so set to MaxInt32/2 to avoid addition overflow. | ||
| 65 | + forceInlineBudget = math.MaxInt32 / 2 | ||
| 61 | ) | 66 | ) |
| 62 | 67 | ||
| 63 | var ( | 68 | var ( |
| @@ -159,6 +164,63 @@ func hotNodesFromCDF(p *pgoir.Profile) (float64, []pgo.NamedCallEdge) { | |||
| 159 | return 0, p.NamedEdgeMap.ByWeight | 164 | return 0, p.NamedEdgeMap.ByWeight |
| 160 | } | 165 | } |
| 161 | 166 | ||
| 167 | +// forceInlineMap contains the function list which want to be force inlined. | ||
| 168 | +var forceInlineMap map[string]struct{} | ||
| 169 | +var forceInlineInitOnce = sync.Once{} | ||
| 170 | + | ||
| 171 | +// initForceInlineFuncList init forceInlineMap which contains force inline functions. | ||
| 172 | +func initForceInlineFuncList() { | ||
| 173 | + switch base.Debug.ForceInline { | ||
| 174 | + case 0: | ||
| 175 | + return | ||
| 176 | + | ||
| 177 | + case 1: | ||
| 178 | + // Add default force inline function list. | ||
| 179 | + // You may need update this list when you update the go version. | ||
| 180 | + forceInlineMap = map[string]struct{}{ | ||
| 181 | + "runtime.heapBitsSetType": {}, | ||
| 182 | + "runtime.userArenaHeapBitsSetSliceType": {}, | ||
| 183 | + | ||
| 184 | + "runtime.writeHeapBits.flush": {}, | ||
| 185 | + "runtime.writeHeapBits.write": {}, | ||
 | |||
| 186 | + "runtime.nextFreeFast": {}, | ||
| 187 | + "runtime.(*mcache).nextFree": {}, | ||
| 188 | + "runtime.(*mcentral).cacheSpan": {}, | ||
| 189 | + "runtime.gcmarknewobject": {}, | ||
| 190 | + "runtime.deductAssistCredit": {}, | ||
| 191 | + "runtime.deductAssistCredit2": {}, | ||
| 192 | + | ||
| 193 | + "runtime.mallocgcSmallScanNoHeader": {}, | ||
| 194 | + | ||
| 195 | + "runtime.mallocgc1": {}, | ||
| 196 | + "runtime.mallocgc2": {}, | ||
🟠 High Priority runtime 包中仅存在 deductAssistCredit，不存在 deductAssistCredit2。该条目永不匹配。 同一段代码其他问题
  不准确？ huang-xiaoquan 5月18日 评论： 5月18日 评论： | |||
| 197 | + "runtime.mallocgc": {}, | ||
| 198 | + | ||
| 199 | + "runtime.scanobject": {}, | ||
| 200 | + "runtime.findObject": {}, | ||
| 201 | + "runtime.greyobject": {}, | ||
| 202 | + "runtime.newobject": {}, | ||
| 203 | + "runtime.bgsweep": {}, | ||
| 204 | + } | ||
| 205 | + | ||
| 206 | + default: | ||
| 207 | + base.Errorf("-forceinline does not support setting to %d", base.Debug.ForceInline) | ||
| 208 | + base.ErrorExit() | ||
| 209 | + } | ||
| 210 | +} | ||
| 211 | + | ||
| 212 | +// isForceInlineFunc determines whether the function needs to be force inlined. | ||
| 213 | +func isForceInlineFunc(fn *ir.Func) bool { | ||
| 214 | + if base.Debug.ForceInline == 0 { | ||
| 215 | + return false | ||
| 216 | + } | ||
| 217 | + forceInlineInitOnce.Do(initForceInlineFuncList) | ||
| 218 | + | ||
| 219 | + // Note: Here use ir.LinkFuncName, not ir.PkgFuncName. | ||
| 220 | + _, ok := forceInlineMap[ir.LinkFuncName(fn)] | ||
| 221 | + return ok | ||
| 222 | +} | ||
| 223 | + | ||
| 162 | // CanInlineFuncs computes whether a batch of functions are inlinable. | 224 | // CanInlineFuncs computes whether a batch of functions are inlinable. |
| 163 | func CanInlineFuncs(funcs []*ir.Func, profile *pgoir.Profile) { | 225 | func CanInlineFuncs(funcs []*ir.Func, profile *pgoir.Profile) { |
| 164 | if profile != nil { | 226 | if profile != nil { |
| @@ -187,6 +249,13 @@ func CanInlineFuncs(funcs []*ir.Func, profile *pgoir.Profile) { | |||
| 187 | // adjusted downwards. If 'verbose' is set, then print a remark where | 249 | // adjusted downwards. If 'verbose' is set, then print a remark where |
| 188 | // we boost the budget due to PGO. | 250 | // we boost the budget due to PGO. |
| 189 | func inlineBudget(fn *ir.Func, profile *pgoir.Profile, relaxed bool, verbose bool) int32 { | 251 | func inlineBudget(fn *ir.Func, profile *pgoir.Profile, relaxed bool, verbose bool) int32 { |
| 252 | + if isForceInlineFunc(fn) { | ||
| 253 | + // Update the budget for force inline function. | ||
| 254 | + if base.Debug.ForceInlineLog > 1 { | ||
| 255 | + fmt.Printf("force-inline enabled increased budget=%v for func=%v\n", forceInlineBudget, ir.PkgFuncName(fn)) | ||
| 256 | + } | ||
| 257 | + return forceInlineBudget | ||
| 258 | + } | ||
| 190 | // Update the budget for profile-guided inlining. | 259 | // Update the budget for profile-guided inlining. |
| 191 | budget := int32(inlineMaxBudget) | 260 | budget := int32(inlineMaxBudget) |
| 192 | if IsPgoHotFunc(fn, profile) { | 261 | if IsPgoHotFunc(fn, profile) { |
| @@ -214,7 +283,8 @@ func CanInline(fn *ir.Func, profile *pgoir.Profile) { | |||
| 214 | } | 283 | } |
| 215 | 284 | ||
| 216 | var reason string // reason, if any, that the function was not inlined | 285 | var reason string // reason, if any, that the function was not inlined |
| 217 | - if base.Flag.LowerM > 1 || logopt.Enabled() { | 286 | + var isForceInlineFn = isForceInlineFunc(fn) |
| 287 | + if base.Flag.LowerM > 1 || logopt.Enabled() || (base.Debug.ForceInlineLog > 1 && isForceInlineFn) { | ||
| 218 | defer func() { | 288 | defer func() { |
| 219 | if reason != "" { | 289 | if reason != "" { |
| 220 | if base.Flag.LowerM > 1 { | 290 | if base.Flag.LowerM > 1 { |
| @@ -223,6 +293,11 @@ func CanInline(fn *ir.Func, profile *pgoir.Profile) { | |||
| 223 | if logopt.Enabled() { | 293 | if logopt.Enabled() { |
| 224 | logopt.LogOpt(fn.Pos(), "cannotInlineFunction", "inline", ir.FuncName(fn), reason) | 294 | logopt.LogOpt(fn.Pos(), "cannotInlineFunction", "inline", ir.FuncName(fn), reason) |
| 225 | } | 295 | } |
| 296 | + if isForceInlineFn { | ||
| 297 | + // Print the reason why the function cannot be force inlined. | ||
| 298 | + fmt.Printf("force-inline cannot inline func=%v: %s at %v\n", | ||
| 299 | + ir.PkgFuncName(fn), reason, ir.Line(fn)) | ||
| 300 | + } | ||
| 226 | } | 301 | } |
| 227 | }() | 302 | }() |
| 228 | } | 303 | } |
| @@ -898,7 +973,7 @@ var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlInde | |||
| 898 | // - the "max cost" limit used to make the decision (which may differ depending on func size) | 973 | // - the "max cost" limit used to make the decision (which may differ depending on func size) |
| 899 | // - the score assigned to this specific callsite | 974 | // - the score assigned to this specific callsite |
| 900 | // - whether the inlined function is "hot" according to PGO. | 975 | // - whether the inlined function is "hot" according to PGO. |
| 901 | -func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCalledOnce bool) (bool, int32, int32, bool) { | 976 | +func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCalledOnce bool, forceInl bool) (bool, int32, int32, bool) { |
| 902 | maxCost := int32(inlineMaxBudget) | 977 | maxCost := int32(inlineMaxBudget) |
| 903 | 978 | ||
| 904 | if bigCaller { | 979 | if bigCaller { |
| @@ -906,6 +981,9 @@ func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller, closureCal | |||
| 906 | // See issue 26546 and 17566. | 981 | // See issue 26546 and 17566. |
| 907 | maxCost = inlineBigFunctionMaxCost | 982 | maxCost = inlineBigFunctionMaxCost |
| 908 | } | 983 | } |
| 984 | + if forceInl { | ||
| 985 | + maxCost = forceInlineBudget | ||
| 986 | + } | ||
| 909 | 987 | ||
| 910 | if callee.ClosureParent != nil { | 988 | if callee.ClosureParent != nil { |
| 911 | maxCost *= 2 // favor inlining closures | 989 | maxCost *= 2 // favor inlining closures |
| @@ -991,8 +1069,21 @@ func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCa | |||
| 991 | return false, 0, false | 1069 | return false, 0, false |
| 992 | } | 1070 | } |
| 993 | 1071 | ||
| 994 | - ok, maxCost, callSiteScore, hot := inlineCostOK(n, callerfn, callee, bigCaller, closureCalledOnce) | 1072 | + forceInl := isForceInlineFunc(callee) |
| 995 | - if !ok { | 1073 | + ok, maxCost, callSiteScore, hot := inlineCostOK(n, callerfn, callee, bigCaller, closureCalledOnce, forceInl) |
| 1074 | + if forceInl { | ||
| 1075 | + if !ok { | ||
| 1076 | + if base.Debug.ForceInlineLog > 0 { | ||
| 1077 | + fmt.Printf("force-inline cannot inline func=%s: cost=%d exceeds budget=%d at %v\n", | ||
| 1078 | + ir.PkgFuncName(callee), callee.Inl.Cost, forceInlineBudget, ir.Line(n)) | ||
| 1079 | + } | ||
| 1080 | + return false, 0, false | ||
| 1081 | + } | ||
| 1082 | + if base.Debug.ForceInlineLog > 0 { | ||
| 1083 | + fmt.Printf("force-inline check allows `%s` inlining `%s` (cost=%d) at %v\n", ir.PkgFuncName(ir.CurFunc), | ||
| 1084 | + ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n)) | ||
| 1085 | + } | ||
| 1086 | + } else if !ok { | ||
| 996 | // callee cost too high for this call site. | 1087 | // callee cost too high for this call site. |
| 997 | if log && logopt.Enabled() { | 1088 | if log && logopt.Enabled() { |
| 998 | logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn), | 1089 | logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn), |
| @@ -0,0 +1,178 @@ | |||
| 1 | +# Test for force inline feature (-d forceinline option) | ||
| 2 | + | ||
| 3 | +env GO111MODULE=on | ||
| 4 | +env GOCACHE=${WORK}${/}gocache | ||
| 5 | +env GOPATH=${WORK}${/}gopath | ||
| 6 | + | ||
| 7 | +# ---------------------------------------------------------------------------------------------------------------------- | ||
| 8 | +# Use -gcflags='-d forceinline=1', check default force inline function list worked. | ||
| 9 | +# | ||
| 10 | + | ||
| 11 | +go build -a -gcflags='all=-d forceinline=1 -d=forceinlinelog=2' . | ||
| 12 | + | ||
| 13 | +stderr -q 'force-inline enabled .* func=runtime.userArenaHeapBitsSetSliceType' | ||
| 14 | +stderr -q 'force-inline check allows `runtime\.\(\*mspan\)\.userArenaNextFree` inlining `runtime.userArenaHeapBitsSetSliceType`' | ||
| 15 | + | ||
| 16 | +stderr -q 'force-inline enabled .* func=runtime.mallocgcSmallScanNoHeader' | ||
| 17 | +stderr -q 'force-inline check allows `runtime.mallocgc` inlining `runtime.mallocgcSmallScanNoHeader`' | ||
| 18 | + | ||
| 19 | +stderr -q 'force-inline enabled .* func=runtime.mallocgc' | ||
| 20 | +stderr -q 'force-inline check allows `runtime.newobject` inlining `runtime.mallocgc`' | ||
| 21 | + | ||
| 22 | +stderr -q 'force-inline enabled .* func=runtime.scanobject' | ||
| 23 | +stderr -q 'force-inline check allows `runtime.gcDrain` inlining `runtime.scanobject`' | ||
| 24 | + | ||
| 25 | +stderr -q 'force-inline enabled .* func=runtime.findObject' | ||
| 26 | +stderr -q 'force-inline check allows `runtime.scanobject` inlining `runtime.findObject`' | ||
| 27 | + | ||
| 28 | +# Check the small functions be inlined. | ||
| 29 | +! stderr -q 'force-inline enabled .* func=test_module/nomain/sub.SmallFunc' | ||
| 30 | + | ||
| 31 | +# Check the big functions not inlined. | ||
| 32 | +! stderr -q 'force-inline enabled .* func=test_module/nomain/sub.BigFunc' | ||
| 33 | + | ||
| 34 | +# ---------------------------------------------------------------------------------------------------------------------- | ||
| 35 | +# Use -gcflags="-d forceinline=0" check disable force inline feature. | ||
| 36 | +# | ||
| 37 | + | ||
| 38 | +go build -a -gcflags='all=-d forceinline=0 -d forceinlinelog=2' . | ||
| 39 | + | ||
| 40 | +! stderr -q 'force-inline enabled' | ||
| 41 | +! stderr -q 'force-inline check allows' | ||
| 42 | + | ||
| 43 | +# ---------------------------------------------------------------------------------------------------------------------- | ||
| 44 | +# Use -gcflags="-d forceinline=2", it is illegal setting value, expect build failed. | ||
| 45 | +# | ||
| 46 | + | ||
| 47 | +! go build -a -gcflags='all=-d forceinline=2 -d forceinlinelog=2' . | ||
| 48 | + | ||
| 49 | +stderr -q '-forceinline does not support setting to 2' | ||
| 50 | + | ||
| 51 | +# ---------------------------------------------------------------------------------------------------------------------- | ||
| 52 | +# Use -gcflags='all=-l -d forceinline=1", all functions cannot be inlined. | ||
| 53 | +# this test is verify the priority of option forceinline is lower than gcflags -l | ||
| 54 | +# | ||
| 55 | + | ||
| 56 | +rm output | ||
| 57 | +go build -a -gcflags='all=-l -d forceinline=1 -d forceinlinelog=2' . | ||
| 58 | + | ||
| 59 | +! stderr -q 'force-inline enabled .* func=runtime.userArenaHeapBitsSetSliceType' | ||
| 60 | +! stderr -q 'force-inline enabled .* func=runtime.mallocgcSmallScanNoHeader' | ||
| 61 | +! stderr -q 'force-inline enabled .* func=runtime.mallocgc' | ||
| 62 | +! stderr -q 'force-inline enabled .* func=runtime.scanobject' | ||
| 63 | +! stderr -q 'force-inline enabled .* func=runtime.findObject' | ||
| 64 | + | ||
| 65 | +# ---------------------------------------------------------------------------------------------------------------------- | ||
| 66 | +# Use -gcflags='all=-d forceinline=1", check the Binary Equivalence (BEP). | ||
| 67 | +# | ||
| 68 | + | ||
| 69 | +go build -a -gcflags='all=-d forceinline=1 -d forceinlinelog=2' -o bin1 . | ||
| 70 | +go build -a -gcflags='all=-d forceinline=1 -d forceinlinelog=2' -o bin2 . | ||
| 71 | + | ||
| 72 | +cmp -q bin1 bin2 | ||
| 73 | + | ||
| 74 | +-- go.mod -- | ||
| 75 | +module test_module | ||
| 76 | + | ||
| 77 | +go 1.24 | ||
| 78 | +-- main.go -- | ||
| 79 | +package main | ||
| 80 | + | ||
| 81 | +import "test_module/nomain/sub" | ||
| 82 | + | ||
| 83 | +func main() { | ||
| 84 | + sub.Test() | ||
| 85 | + sub.Test2() | ||
| 86 | + test() | ||
| 87 | +} | ||
| 88 | + | ||
| 89 | +func test() { | ||
| 90 | + sub.SmallFunc() | ||
| 91 | + sub.BigFunc() | ||
| 92 | +} | ||
| 93 | +-- nomain/sub/sub.go -- | ||
| 94 | +package sub | ||
| 95 | + | ||
| 96 | +func Test() { | ||
| 97 | + SmallFunc() | ||
| 98 | + BigFunc() | ||
| 99 | +} | ||
| 100 | + | ||
| 101 | +func SmallFunc() { | ||
| 102 | + println(1) | ||
| 103 | +} | ||
| 104 | + | ||
| 105 | +func BigFunc() { | ||
| 106 | + canInlineFunc() | ||
| 107 | + canInlineFunc() | ||
| 108 | + canInlineFunc() | ||
| 109 | + canInlineFunc() | ||
| 110 | + canInlineFunc() | ||
| 111 | + canInlineFunc() | ||
| 112 | + canInlineFunc() | ||
| 113 | + canInlineFunc() | ||
| 114 | + canInlineFunc() | ||
| 115 | + canInlineFunc() | ||
| 116 | + canInlineFunc() | ||
| 117 | + canInlineFunc() | ||
| 118 | + canInlineFunc() | ||
| 119 | + canInlineFunc() | ||
| 120 | + canInlineFunc() | ||
| 121 | + canInlineFunc() | ||
| 122 | + canInlineFunc() | ||
| 123 | + canInlineFunc() | ||
| 124 | + canInlineFunc() | ||
| 125 | + canInlineFunc() | ||
| 126 | + canInlineFunc() | ||
| 127 | + canInlineFunc() | ||
| 128 | + canInlineFunc() | ||
| 129 | + canInlineFunc() | ||
| 130 | + canInlineFunc() | ||
| 131 | + canInlineFunc() | ||
| 132 | + canInlineFunc() | ||
| 133 | + canInlineFunc() | ||
| 134 | + canInlineFunc() | ||
| 135 | + canInlineFunc() | ||
| 136 | + canInlineFunc() | ||
| 137 | + canInlineFunc() | ||
| 138 | + canInlineFunc() | ||
| 139 | + canInlineFunc() | ||
| 140 | + canInlineFunc() | ||
| 141 | + canInlineFunc() | ||
| 142 | + canInlineFunc() | ||
| 143 | + canInlineFunc() | ||
| 144 | + canInlineFunc() | ||
| 145 | + canInlineFunc() | ||
| 146 | + canInlineFunc() | ||
| 147 | + canInlineFunc() | ||
| 148 | + canInlineFunc() | ||
| 149 | + canInlineFunc() | ||
| 150 | + canInlineFunc() | ||
| 151 | +} | ||
| 152 | + | ||
| 153 | +// This function's inline cost is 70 | ||
| 154 | +func canInlineFunc() { | ||
| 155 | + println(1, 2, 3, 4, 5, 6, 7, 8, 9) | ||
| 156 | + println(1, 2, 3, 4, 5, 6, 7, 8, 9) | ||
| 157 | + println(1, 2, 3, 4, 5, 6, 7, 8, 9) | ||
| 158 | + println(1, 2, 3, 4, 5, 6, 7, 8, 9) | ||
| 159 | + println(1, 2, 3, 4, 5, 6, 7, 8, 9) | ||
| 160 | + println(1, 2, 3, 4, 5, 6, 7, 8, 9) | ||
| 161 | + println(1, 2, 3, 4, 5, 6, 7, 8, 9) | ||
| 162 | +} | ||
| 163 | + | ||
| 164 | +func Test2() { | ||
| 165 | + useDeferFunc() | ||
| 166 | + markNoInlineFunc() | ||
| 167 | +} | ||
| 168 | + | ||
| 169 | +func useDeferFunc() { | ||
| 170 | + defer func() { | ||
| 171 | + print(1) | ||
| 172 | + }() | ||
| 173 | +} | ||
| 174 | + | ||
| 175 | +//go:noinline | ||
| 176 | +func markNoInlineFunc() { | ||
| 177 | + println(1) | ||
| 178 | +} | ||
🟠 High Priority
Go 1.24.6 运行时中不存在 heapBitsSetType 函数。相关功能已重构为 heapSetTypeNoHeader / heapSetTypeSmallHeader / heapSetTypeLarge。该条目永不会匹配任何函数，导致此强制内联目标失效。
同一段代码其他问题
forceInlineMap 中 runtime.writeHeapBits.flush 和 .write 不存在：运行时中不存在 writeHeapBits 类型。writeHeapBitsSmall 是 *mspan 的方法，而非独立类型。flush 和 write 方法均不存在。这两个条目永不会匹配。缩进不一致：使用空格代替制表符：第 182-183 行使用空格缩进，而 Go 惯用制表符。第 183 行还包含尾随空格。不影响编译但违反 Go 代码风格。