Star4
36
Code
Code
Issues12
Pull Requests34
Discussion
Wiki
Analysis
Star4
36
Merged
[Backport] Align tight and hot loops without a max-skip limit #393
haibaooAdd on Aug 17
[Backport] Align tight and hot loops without a max-skip limit #393
Merged
haibaooAdd on Aug 17
Merged fromqq_56722097/gccbackport-align-tight-loopsintoopeneuler/gccmaster
Total: 9 files changed+236-6
Mgcc/config/i386/i386-features.cc+197-1
@@ -2163,6 +2163,203 @@ make_pass_insert_endbr_and_patchable_area (gcc::context *ctxt)
2163 return new pass_insert_endbr_and_patchable_area (ctxt);2163 return new pass_insert_endbr_and_patchable_area (ctxt);
2164}2164}
2165 2165 
2166+/* When a hot loop can be fit into one cacheline,
2167+ force align the loop without considering the max skip. */
2168+static void
2169+ix86_align_loops ()
2170+{
2171+ basic_block bb;
2172+ 
2173+ /* Don't do this when we don't know cache line size. */
2174+ if (ix86_cost->prefetch_block == 0)
2175+ return;
2176+ 
2177+ loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
2178+ profile_count count_threshold = cfun->cfg->count_max.apply_scale
2179+ (1, param_align_threshold);
2180+ FOR_EACH_BB_FN (bb, cfun)
2181+ {
2182+ rtx_insn *label = BB_HEAD (bb);
2183+ bool has_fallthru = 0;
2184+ edge e;
2185+ edge_iterator ei;
2186+ 
2187+ if (!LABEL_P (label))
2188+ continue;
2189+ 
2190+ profile_count fallthru_count = profile_count::zero ();
2191+ profile_count branch_count = profile_count::zero ();
2192+ 
2193+ FOR_EACH_EDGE (e, ei, bb->preds)
2194+ {
2195+ if (e->flags & EDGE_FALLTHRU)
2196+ has_fallthru = 1, fallthru_count += e->count ();
2197+ else
2198+ branch_count += e->count ();
2199+ }
2200+ 
2201+ if (!fallthru_count.initialized_p () || !branch_count.initialized_p ())
2202+ continue;
2203+ 
2204+ if (bb->loop_father
2205+ && bb->loop_father->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
2206+ && (has_fallthru
2207+ ? (!(single_succ_p (bb)
2208+ && single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun))
2209+ && optimize_bb_for_speed_p (bb)
2210+ && branch_count + fallthru_count > count_threshold
2211+ && (branch_count
2212+ > fallthru_count.apply_scale
2213+ (param_align_loop_iterations, 1)))
2214+ /* In case there'no fallthru for the loop.
2215+ Nops inserted won't be executed. */
2216+ : (branch_count > count_threshold
2217+ || (bb->count > bb->prev_bb->count.apply_scale (10, 1)
2218+ && (bb->prev_bb->count
2219+ <= ENTRY_BLOCK_PTR_FOR_FN (cfun)
2220+ ->count.apply_scale (1, 2))))))
2221+ {
2222+ rtx_insn* insn, *end_insn;
2223+ HOST_WIDE_INT size = 0;
2224+ bool padding_p = true;
2225+ basic_block tbb = bb;
2226+ unsigned cond_branch_num = 0;
2227+ bool detect_tight_loop_p = false;
2228+ 
2229+ for (unsigned int i = 0; i != bb->loop_father->num_nodes;
2230+ i++, tbb = tbb->next_bb)
2231+ {
2232+ /* Only handle continuous cfg layout. */
2233+ if (bb->loop_father != tbb->loop_father)
2234+ {
2235+ padding_p = false;
2236+ break;
2237+ }
2238+ 
2239+ FOR_BB_INSNS (tbb, insn)
2240+ {
2241+ if (!NONDEBUG_INSN_P (insn))
2242+ continue;
2243+ size += ix86_min_insn_size (insn);
2244+ 
2245+ /* We don't know size of inline asm.
2246+ Don't align loop for call. */
2247+ if (asm_noperands (PATTERN (insn)) >= 0
2248+ || CALL_P (insn))
2249+ {
2250+ size = -1;
2251+ break;
2252+ }
2253+ }
2254+ 
2255+ if (size == -1 || size > ix86_cost->prefetch_block)
2256+ {
2257+ padding_p = false;
2258+ break;
2259+ }
2260+ 
2261+ FOR_EACH_EDGE (e, ei, tbb->succs)
2262+ {
2263+ /* It could be part of the loop. */
2264+ if (e->dest == bb)
2265+ {
2266+ detect_tight_loop_p = true;
2267+ break;
2268+ }
2269+ }
2270+ 
2271+ if (detect_tight_loop_p)
2272+ break;
2273+ 
2274+ end_insn = BB_END (tbb);
2275+ if (JUMP_P (end_insn))
2276+ {
2277+ /* For decoded icache:
2278+ 1. Up to two branches are allowed per Way.
2279+ 2. A non-conditional branch is the last micro-op in a Way.
2280+ */
2281+ if (onlyjump_p (end_insn)
2282+ && (any_uncondjump_p (end_insn)
2283+ || single_succ_p (tbb)))
2284+ {
2285+ padding_p = false;
2286+ break;
2287+ }
2288+ else if (++cond_branch_num >= 2)
2289+ {
2290+ padding_p = false;
2291+ break;
2292+ }
2293+ }
2294+ 
2295+ }
2296+ 
2297+ if (padding_p && detect_tight_loop_p)
2298+ {
2299+ emit_insn_before (gen_max_skip_align (GEN_INT (ceil_log2 (size)),
2300+ GEN_INT (0)), label);
2301+ /* End of function. */
2302+ if (!tbb || tbb == EXIT_BLOCK_PTR_FOR_FN (cfun))
2303+ break;
2304+ /* Skip bb which already fits into one cacheline. */
2305+ bb = tbb;
2306+ }
2307+ }
2308+ }
2309+ 
2310+ loop_optimizer_finalize ();
2311+ free_dominance_info (CDI_DOMINATORS);
2312+}
2313+ 
2314+namespace {
2315+ 
2316+const pass_data pass_data_align_tight_loops =
2317+{
2318+ RTL_PASS, /* type */
2319+ "align_tight_loops", /* name */
2320+ OPTGROUP_NONE, /* optinfo_flags */
2321+ TV_MACH_DEP, /* tv_id */
2322+ 0, /* properties_required */
2323+ 0, /* properties_provided */
2324+ 0, /* properties_destroyed */
2325+ 0, /* todo_flags_start */
2326+ 0, /* todo_flags_finish */
2327+};
2328+ 
2329+class pass_align_tight_loops : public rtl_opt_pass
2330+{
2331+public:
2332+ pass_align_tight_loops (gcc::context *ctxt)
2333+ : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
2334+ {}
2335+ 
2336+ /* opt_pass methods: */
2337+ bool gate (function *) final override
2338+ {
2339+ return TARGET_ALIGN_TIGHT_LOOPS
2340+ && optimize
2341+ && optimize_function_for_speed_p (cfun);
2342+ }
2343+ 
2344+ unsigned int execute (function *) final override
2345+ {
2346+ timevar_push (TV_MACH_DEP);
2347+#ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
2348+ ix86_align_loops ();
2349+#endif
2350+ timevar_pop (TV_MACH_DEP);
2351+ return 0;
2352+ }
2353+}; // class pass_align_tight_loops
2354+ 
2355+} // anon namespace
2356+ 
2357+rtl_opt_pass *
2358+make_pass_align_tight_loops (gcc::context *ctxt)
2359+{
2360+ return new pass_align_tight_loops (ctxt);
2361+}
2362+ 
2166/* At entry of the nearest common dominator for basic blocks with2363/* At entry of the nearest common dominator for basic blocks with
2167 conversions/rcp/sqrt/rsqrt/round, generate a single2364 conversions/rcp/sqrt/rsqrt/round, generate a single
2168 vxorps %xmmN, %xmmN, %xmmN2365 vxorps %xmmN, %xmmN, %xmmN
@@ -2965,4 +3162,3 @@ ix86_generate_version_dispatcher_body (void *node_p)
2965 return resolver_decl;3162 return resolver_decl;
2966}3163}
2967 3164 
2968- 
Mgcc/config/i386/i386-passes.def+3-0
@@ -31,5 +31,8 @@ along with GCC; see the file COPYING3. If not see
31 INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */);31 INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */);
32 32 
33 INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_insert_endbr_and_patchable_area);33 INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_insert_endbr_and_patchable_area);
34+ /* pass_align_tight_loops must be after pass_insert_endbr_and_patchable_area.
35+ PR116174. */
36+ INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_align_tight_loops);
34 37 
35 INSERT_PASS_AFTER (pass_combine, 1, pass_remove_partial_avx_dependency);38 INSERT_PASS_AFTER (pass_combine, 1, pass_remove_partial_avx_dependency);
Mgcc/config/i386/i386-protos.h+1-0
@@ -397,6 +397,7 @@ extern rtl_opt_pass *make_pass_insert_vzeroupper (gcc::context *);
397extern rtl_opt_pass *make_pass_stv (gcc::context *);397extern rtl_opt_pass *make_pass_stv (gcc::context *);
398extern rtl_opt_pass *make_pass_insert_endbr_and_patchable_area398extern rtl_opt_pass *make_pass_insert_endbr_and_patchable_area
399 (gcc::context *);399 (gcc::context *);
400+extern rtl_opt_pass *make_pass_align_tight_loops (gcc::context *);
400extern rtl_opt_pass *make_pass_remove_partial_avx_dependency401extern rtl_opt_pass *make_pass_remove_partial_avx_dependency
401 (gcc::context *);402 (gcc::context *);
402 403 
Mgcc/config/i386/i386.cc+2-1
@@ -21793,7 +21793,8 @@ ix86_avoid_jump_mispredicts (void)
21793 if (dump_file)21793 if (dump_file)
21794 fprintf (dump_file, "Padding insn %i by %i bytes!\n",21794 fprintf (dump_file, "Padding insn %i by %i bytes!\n",
21795 INSN_UID (insn), padsize);21795 INSN_UID (insn), padsize);
21796- emit_insn_before (gen_pad (GEN_INT (padsize)), insn);21796+ emit_insn_before (gen_max_skip_align (GEN_INT (4),
21797+ GEN_INT (padsize)), insn);
21797 }21798 }
21798 }21799 }
21799}21800}
Mgcc/config/i386/i386.h+2-0
@@ -369,6 +369,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
369#define TARGET_INTER_UNIT_CONVERSIONS \369#define TARGET_INTER_UNIT_CONVERSIONS \
370 ix86_tune_features[X86_TUNE_INTER_UNIT_CONVERSIONS]370 ix86_tune_features[X86_TUNE_INTER_UNIT_CONVERSIONS]
371#define TARGET_FOUR_JUMP_LIMIT ix86_tune_features[X86_TUNE_FOUR_JUMP_LIMIT]371#define TARGET_FOUR_JUMP_LIMIT ix86_tune_features[X86_TUNE_FOUR_JUMP_LIMIT]
372+#define TARGET_ALIGN_TIGHT_LOOPS \
373+ ix86_tune_features[X86_TUNE_ALIGN_TIGHT_LOOPS]
372#define TARGET_SCHEDULE ix86_tune_features[X86_TUNE_SCHEDULE]374#define TARGET_SCHEDULE ix86_tune_features[X86_TUNE_SCHEDULE]
373#define TARGET_USE_BT ix86_tune_features[X86_TUNE_USE_BT]375#define TARGET_USE_BT ix86_tune_features[X86_TUNE_USE_BT]
374#define TARGET_USE_INCDEC ix86_tune_features[X86_TUNE_USE_INCDEC]376#define TARGET_USE_INCDEC ix86_tune_features[X86_TUNE_USE_INCDEC]
Mgcc/config/i386/i386.md+7-4
@@ -15539,16 +15539,19 @@
15539 (set_attr "length_immediate" "0")15539 (set_attr "length_immediate" "0")
15540 (set_attr "modrm" "0")])15540 (set_attr "modrm" "0")])
15541 15541 
15542-;; Pad to 16-byte boundary, max skip in op0. Used to avoid15542+;; Pad to 1 << op0 byte boundary, max skip in op1. Used to avoid
15543;; branch prediction penalty for the third jump in a 16-byte15543;; branch prediction penalty for the third jump in a 16-byte
15544;; block on K8.15544;; block on K8.
15545+;; Also it's used to align tight loops which can be fix into 1 cacheline.
15546+;; It can help code prefetch and reduce DSB miss.
15545 15547 
15546-(define_insn "pad"15548+(define_insn "max_skip_align"
15547- [(unspec_volatile [(match_operand 0)] UNSPECV_ALIGN)]15549+ [(unspec_volatile [(match_operand 0) (match_operand 1)] UNSPECV_ALIGN)]
15548 ""15550 ""
15549{15551{
15550#ifdef ASM_OUTPUT_MAX_SKIP_ALIGN15552#ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
15551- ASM_OUTPUT_MAX_SKIP_ALIGN (asm_out_file, 4, (int)INTVAL (operands[0]));15553+ ASM_OUTPUT_MAX_SKIP_ALIGN (asm_out_file, (int)INTVAL (operands[0]),
15554+ (int)INTVAL (operands[1]));
15552#else15555#else
15553 /* It is tempting to use ASM_OUTPUT_ALIGN here, but we don't want to do that.15556 /* It is tempting to use ASM_OUTPUT_ALIGN here, but we don't want to do that.
15554 The align insn is used to avoid 3 jump instructions in the row to improve15557 The align insn is used to avoid 3 jump instructions in the row to improve
Mgcc/config/i386/x86-tune.def+3-0
@@ -224,6 +224,9 @@ DEF_TUNE (X86_TUNE_FOUR_JUMP_LIMIT, "four_jump_limit",
224 m_PPRO | m_P4_NOCONA | m_BONNELL | m_SILVERMONT | m_KNL | m_KNM224 m_PPRO | m_P4_NOCONA | m_BONNELL | m_SILVERMONT | m_KNL | m_KNM
225 | m_GOLDMONT | m_GOLDMONT_PLUS | m_INTEL | m_ATHLON_K8 | m_AMDFAM10)225 | m_GOLDMONT | m_GOLDMONT_PLUS | m_INTEL | m_ATHLON_K8 | m_AMDFAM10)
226 226 
227+/* X86_TUNE_ALIGN_TIGHT_LOOPS: if false, tight loops are not aligned. */
228+DEF_TUNE (X86_TUNE_ALIGN_TIGHT_LOOPS, "align_tight_loops", m_C86_4G_M7)
229+ 
227/*****************************************************************************/230/*****************************************************************************/
228/* Integer instruction selection tuning */231/* Integer instruction selection tuning */
229/*****************************************************************************/232/*****************************************************************************/
Agcc/testsuite/gcc.target/i386/align-tight-loop-generic.c+11-0
@@ -0,0 +1,11 @@
1+/* { dg-do compile { target { *-*-linux* && lp64 } } } */
2+/* { dg-options "-O2 -g0 -fcf-protection=branch -mtune=generic" } */
3+/* { dg-final { scan-assembler-not "\\.p2align 5" } } */
4+ 
5+char *
6+foo (char *dest, const char *src)
7+{
8+ while ((*dest++ = *src++) != '\0')
9+ /* nothing */;
10+ return --dest;
11+}
Agcc/testsuite/gcc.target/i386/pr116174.c+10-0
@@ -0,0 +1,10 @@
1+/* { dg-do compile { target { *-*-linux* && lp64 } } } */
2+/* { dg-options "-O2 -g0 -fcf-protection=branch -mtune=c86-4g-m7" } */
3+/* { dg-final { scan-assembler "endbr64\\n\\t\\.p2align 5" } } */
4+char *
5+foo (char *dest, const char *src)
6+{
7+ while ((*dest++ = *src++) != '\0')
8+ /* nothing */;
9+ return --dest;
10+}