[Backport] ira: Consider save/restore costs of callee-save registers-gcc-AtomGit

Mgcc/config/aarch64/aarch64.cc+136-4

	@@ -5359,10 +5359,23 @@ aarch64_hard_regno_call_part_clobbered (unsigned int abi_id,
5359	{	5359	{
5360	if (FP_REGNUM_P (regno) && abi_id != ARM_PCS_SVE)	5360	if (FP_REGNUM_P (regno) && abi_id != ARM_PCS_SVE)
5361	{	5361	{
5362	- poly_int64 per_register_size = GET_MODE_SIZE (mode);	5362	+ poly_int64 per_register_size;
5363	- unsigned int nregs = hard_regno_nregs (regno, mode);	5363	+ if (aarch64_sve_mode_p (mode))
5364	- if (nregs > 1)	5364	+ /* SVE instructions operate on the full SVE register, even for
5365	- per_register_size = exact_div (per_register_size, nregs);	5365	+ partial modes like VNx2SF whose GET_MODE_SIZE is only 8 bytes
		5366	+ under -msve-vector-bits=128. The elements of such partial
		5367	+ modes are strided across the register and can lie outside the
		5368	+ callee-preserved low 64 bits. Compare against the full SVE
		5369	+ vector size so that V8-V15 are correctly recognised as
		5370	+ partially clobbered for any SVE mode under AAPCS64/AAPCS_SIMD. */
		5371	+ per_register_size = BYTES_PER_SVE_VECTOR;
		5372	+ else
		5373	+ {
		5374	+ per_register_size = GET_MODE_SIZE (mode);
		5375	+ unsigned int nregs = hard_regno_nregs (regno, mode);
		5376	+ if (nregs > 1)
		5377	+ per_register_size = exact_div (per_register_size, nregs);
		5378	+ }
5366	if (abi_id == ARM_PCS_SIMD \|\| abi_id == ARM_PCS_TLSDESC)	5379	if (abi_id == ARM_PCS_SIMD \|\| abi_id == ARM_PCS_TLSDESC)
5367	return maybe_gt (per_register_size, 16);	5380	return maybe_gt (per_register_size, 16);
5368	return maybe_gt (per_register_size, 8);	5381	return maybe_gt (per_register_size, 8);
	@@ -17785,6 +17798,119 @@ aarch64_memory_move_cost (machine_mode mode, reg_class_t rclass_i, bool in)
17785	: aarch64_tune_params.memmov_cost.store_int);	17798	: aarch64_tune_params.memmov_cost.store_int);
17786	}	17799	}
17787		17800
		17801	+/* CALLEE_SAVED_REGS is the set of callee-saved registers that the
		17802	+ RA has already decided to use. Return the total number of registers
		17803	+ in class RCLASS that need to be saved and restored, including the
		17804	+ frame link registers. */
		17805	+static int
		17806	+aarch64_count_saves (const HARD_REG_SET &callee_saved_regs,
		17807	+ enum reg_class rclass)
		17808	+{
		17809	+ auto saved_gprs = callee_saved_regs & reg_class_contents[rclass];
		17810	+ auto nregs = hard_reg_set_popcount (saved_gprs);
		17811	+
		17812	+ if (TEST_HARD_REG_BIT (reg_class_contents[rclass], LR_REGNUM))
		17813	+ {
		17814	+ if (aarch64_needs_frame_chain ())
		17815	+ nregs += 2;
		17816	+ else if (!crtl->is_leaf \|\| df_regs_ever_live_p (LR_REGNUM))
		17817	+ nregs += 1;
		17818	+ }
		17819	+ return nregs;
		17820	+}
		17821	+
		17822	+/* CALLEE_SAVED_REGS is the set of callee-saved registers that the
		17823	+ RA has already decided to use. Return the total number of registers
		17824	+ that need to be saved above the hard frame pointer, including the
		17825	+ frame link registers. */
		17826	+static int
		17827	+aarch64_count_above_hard_fp_saves (const HARD_REG_SET &callee_saved_regs)
		17828	+{
		17829	+ /* FP and Advanced SIMD registers are saved above the frame pointer
		17830	+ but SVE registers are saved below it. */
		17831	+ if (known_le (GET_MODE_SIZE (aarch64_reg_save_mode (V8_REGNUM)), 16U))
		17832	+ return aarch64_count_saves (callee_saved_regs, POINTER_AND_FP_REGS);
		17833	+ return aarch64_count_saves (callee_saved_regs, POINTER_REGS);
		17834	+}
		17835	+
		17836	+/* Implement TARGET_CALLEE_SAVE_COST. */
		17837	+static int
		17838	+aarch64_callee_save_cost (spill_cost_type spill_type, unsigned int regno,
		17839	+ machine_mode mode, unsigned int nregs, int mem_cost,
		17840	+ const HARD_REG_SET &callee_saved_regs,
		17841	+ bool existing_spill_p)
		17842	+{
		17843	+ /* If we've already committed to saving an odd number of GPRs, assume that
		17844	+ saving one more will involve turning an STR into an STP and an LDR
		17845	+ into an LDP. This should still be more expensive than not spilling
		17846	+ (meaning that the minimum cost is 1), but it should usually be cheaper
		17847	+ than a separate store or load. */
		17848	+ if (GP_REGNUM_P (regno)
		17849	+ && nregs == 1
		17850	+ && (aarch64_count_saves (callee_saved_regs, GENERAL_REGS) & 1))
		17851	+ return 1;
		17852	+
		17853	+ /* Similarly for saving FP registers, if we only need to save the low
		17854	+ 64 bits. (We can also use STP/LDP instead of STR/LDR for Q registers,
		17855	+ but that is less likely to be a saving.) */
		17856	+ if (FP_REGNUM_P (regno)
		17857	+ && nregs == 1
		17858	+ && known_eq (GET_MODE_SIZE (aarch64_reg_save_mode (regno)), 8U)
		17859	+ && (aarch64_count_saves (callee_saved_regs, FP_REGS) & 1))
		17860	+ return 1;
		17861	+
		17862	+ /* If this would be the first register that we save, add the cost of
		17863	+ allocating or deallocating the frame. For GPR, FPR, and Advanced SIMD
		17864	+ saves, the allocation and deallocation can be folded into the save and
		17865	+ restore. */
		17866	+ if (!existing_spill_p
		17867	+ && !GP_REGNUM_P (regno)
		17868	+ && !(FP_REGNUM_P (regno)
		17869	+ && known_le (GET_MODE_SIZE (aarch64_reg_save_mode (regno)), 16U)))
		17870	+ return default_callee_save_cost (spill_type, regno, mode, nregs, mem_cost,
		17871	+ callee_saved_regs, existing_spill_p);
		17872	+
		17873	+ return mem_cost;
		17874	+}
		17875	+
		17876	+/* Implement TARGET_FRAME_ALLOCATION_COST. */
		17877	+static int
		17878	+aarch64_frame_allocation_cost (frame_cost_type,
		17879	+ const HARD_REG_SET &callee_saved_regs)
		17880	+{
		17881	+ /* The intention is to model the relative costs of different approaches
		17882	+ to storing data on the stack, rather than to model the cost of saving
		17883	+ data vs not saving it. This means that we should return 0 if:
		17884	+
		17885	+ - any frame is going to be allocated with:
		17886	+
		17887	+ stp x29, x30, [sp, #-...]!
		17888	+
		17889	+ to create a frame link.
		17890	+
		17891	+ - any frame is going to be allocated with:
		17892	+
		17893	+ str x30, [sp, #-...]!
		17894	+
		17895	+ to save the link register.
		17896	+
		17897	+ In both cases, the allocation and deallocation instructions are the
		17898	+ same however we store data to the stack. (In the second case, the STR
		17899	+ could be converted to an STP by saving an extra call-preserved register,
		17900	+ but that is modeled by aarch64_callee_save_cost.)
		17901	+
		17902	+ In other cases, assume that a frame would need to be allocated with a
		17903	+ separate subtraction and deallocated with a separate addition. Saves
		17904	+ of call-clobbered registers can then reclaim this cost using a
		17905	+ predecrement store and a postincrement load.
		17906	+
		17907	+ For simplicity, give this addition or subtraction the same cost as
		17908	+ a GPR move. We could parameterize this if necessary. */
		17909	+ if (aarch64_count_above_hard_fp_saves (callee_saved_regs) == 0)
		17910	+ return aarch64_tune_params.regmove_cost->GP2GP;
		17911	+ return 0;
		17912	+}
		17913	+
17788	/* Implement TARGET_INIT_BUILTINS. */	17914	/* Implement TARGET_INIT_BUILTINS. */
17789	static void	17915	static void
17790	aarch64_init_builtins ()	17916	aarch64_init_builtins ()
	@@ -31834,6 +31960,12 @@ aarch64_libgcc_floating_mode_supported_p
31834	#undef TARGET_MEMORY_MOVE_COST	31960	#undef TARGET_MEMORY_MOVE_COST
31835	#define TARGET_MEMORY_MOVE_COST aarch64_memory_move_cost	31961	#define TARGET_MEMORY_MOVE_COST aarch64_memory_move_cost
31836		31962
		31963	+#undef TARGET_CALLEE_SAVE_COST
		31964	+#define TARGET_CALLEE_SAVE_COST aarch64_callee_save_cost
		31965	+
		31966	+#undef TARGET_FRAME_ALLOCATION_COST
		31967	+#define TARGET_FRAME_ALLOCATION_COST aarch64_frame_allocation_cost
		31968	+
31837	#undef TARGET_MIN_DIVISIONS_FOR_RECIP_MUL	31969	#undef TARGET_MIN_DIVISIONS_FOR_RECIP_MUL
31838	#define TARGET_MIN_DIVISIONS_FOR_RECIP_MUL aarch64_min_divisions_for_recip_mul	31970	#define TARGET_MIN_DIVISIONS_FOR_RECIP_MUL aarch64_min_divisions_for_recip_mul
31839		31971

Mgcc/config/i386/i386.cc+24-0

	@@ -19493,6 +19493,28 @@ ix86_class_likely_spilled_p (reg_class_t rclass)
19493	return false;	19493	return false;
19494	}	19494	}
19495		19495
		19496	+/* Implement TARGET_CALLEE_SAVE_COST. */
		19497	+
		19498	+static int
		19499	+ix86_callee_save_cost (spill_cost_type, unsigned int hard_regno, machine_mode,
		19500	+ unsigned int, int mem_cost, const HARD_REG_SET &, bool)
		19501	+{
		19502	+ /* Account for the fact that push and pop are shorter and do their
		19503	+ own allocation and deallocation. */
		19504	+ if (GENERAL_REGNO_P (hard_regno))
		19505	+ {
		19506	+ /* push is 1 byte while typical spill is 4-5 bytes.
		19507	+ ??? We probably should adjust size costs accordingly.
		19508	+ Costs are relative to reg-reg move that has 2 bytes for 32bit
		19509	+ and 3 bytes otherwise. Be sure that no cost table sets cost
		19510	+ to 2, so we end up with 0. */
		19511	+ if (mem_cost <= 2 \|\| optimize_function_for_size_p (cfun))
		19512	+ return 1;
		19513	+ return mem_cost - 2;
		19514	+ }
		19515	+ return mem_cost;
		19516	+}
		19517	+
19496	/* Return true if a set of DST by the expression SRC should be allowed.	19518	/* Return true if a set of DST by the expression SRC should be allowed.
19497	This prevents complex sets of likely_spilled hard regs before reload. */	19519	This prevents complex sets of likely_spilled hard regs before reload. */
19498		19520
	@@ -24683,6 +24705,8 @@ ix86_libgcc_floating_mode_supported_p
24683	#define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class	24705	#define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
24684	#undef TARGET_CLASS_LIKELY_SPILLED_P	24706	#undef TARGET_CLASS_LIKELY_SPILLED_P
24685	#define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p	24707	#define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
		24708	+#undef TARGET_CALLEE_SAVE_COST
		24709	+#define TARGET_CALLEE_SAVE_COST ix86_callee_save_cost
24686		24710
24687	#undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST	24711	#undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
24688	#define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \	24712	#define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \

Mgcc/doc/tm.texi+69-0

	@@ -6784,6 +6784,75 @@ value to the result of that function. The arguments to that function
6784	are the same as to this target hook.	6784	are the same as to this target hook.
6785	@end deftypefn	6785	@end deftypefn
6786		6786
		6787	+@deftypefn {Target Hook} int TARGET_CALLEE_SAVE_COST (spill_cost_type @var{cost_type}, unsigned int @var{hard_regno}, machine_mode @var{mode}, unsigned int @var{nregs}, int @var{mem_cost}, const HARD_REG_SET @var{&allocated_callee_regs}, bool @var{existing_spills_p})
		6788	+Return the one-off cost of saving or restoring callee-saved registers
		6789	+(also known as call-preserved registers or non-volatile registers).
		6790	+The parameters are as follows:
		6791	+
		6792	+@itemize
		6793	+@item
		6794	+@var{cost_type} is @samp{spill_cost_type::SAVE} for saving a register
		6795	+and @samp{spill_cost_type::RESTORE} for restoring a register.
		6796	+
		6797	+@item
		6798	+@var{hard_regno} and @var{mode} represent the whole register that
		6799	+the register allocator is considering using; of these,
		6800	+@var{nregs} registers are fully or partially callee-saved.
		6801	+
		6802	+@item
		6803	+@var{mem_cost} is the normal cost for storing (for saves)
		6804	+or loading (for restores) the @var{nregs} registers.
		6805	+
		6806	+@item
		6807	+@var{allocated_callee_regs} is the set of callee-saved registers
		6808	+that are already in use.
		6809	+
		6810	+@item
		6811	+@var{existing_spills_p} is true if the register allocator has
		6812	+already decided to spill registers to memory.
		6813	+@end itemize
		6814	+
		6815	+If @var{existing_spills_p} is false, the cost of a save should account
		6816	+for frame allocations in a way that is consistent with
		6817	+@code{TARGET_FRAME_ALLOCATION_COST}'s handling of allocations for spills.
		6818	+Similarly, the cost of a restore should then account for frame deallocations
		6819	+in a way that is consistent with @code{TARGET_FRAME_ALLOCATION_COST}'s
		6820	+handling of deallocations.
		6821	+
		6822	+Note that this hook should not attempt to apply a frequency scale
		6823	+to the cost: it is the caller's responsibility to do that where
		6824	+appropriate.
		6825	+
		6826	+The default implementation returns @var{mem_cost}, plus the allocation
		6827	+or deallocation cost returned by @code{TARGET_FRAME_ALLOCATION_COST},
		6828	+where appropriate.
		6829	+@end deftypefn
		6830	+
		6831	+@deftypefn {Target Hook} int TARGET_FRAME_ALLOCATION_COST (frame_cost_type @var{cost_type}, const HARD_REG_SET @var{&allocated_callee_regs})
		6832	+Return the cost of allocating or deallocating a frame for the sake of
		6833	+a spill; @var{cost_type} chooses between allocation and deallocation.
		6834	+The term ``spill'' here includes both forcing a pseudo register to memory
		6835	+and using caller-saved registers for pseudo registers that are live across
		6836	+a call.
		6837	+
		6838	+This hook is only called if the register allocator has not so far
		6839	+decided to spill. The allocator may have decided to use callee-saved
		6840	+registers; if so, @var{allocated_callee_regs} is the set of callee-saved
		6841	+registers that the allocator has used. There might also be other reasons
		6842	+why a stack frame is already needed; for example, @samp{get_frame_size ()}
		6843	+might be nonzero, or the target might already require a frame for
		6844	+target-specific reasons.
		6845	+
		6846	+When the register allocator uses this hook to cost spills, it also uses
		6847	+@code{TARGET_CALLEE_SAVE_COST} to cost new callee-saved registers, passing
		6848	+@samp{false} as the @var{existing_spills_p} argument. The intention is to
		6849	+allow the target to apply an apples-for-apples comparison between the
		6850	+cost of using callee-saved registers and using spills in cases where the
		6851	+allocator has not yet committed to using both strategies.
		6852	+
		6853	+The default implementation returns 0.
		6854	+@end deftypefn
		6855	+
6787	@defmac BRANCH_COST (@var{speed_p}, @var{predictable_p})	6856	@defmac BRANCH_COST (@var{speed_p}, @var{predictable_p})
6788	A C expression for the cost of a branch instruction. A value of 1 is	6857	A C expression for the cost of a branch instruction. A value of 1 is
6789	the default; other values are interpreted relative to that. Parameter	6858	the default; other values are interpreted relative to that. Parameter

Mgcc/doc/tm.texi.in+4-0

	@@ -4467,6 +4467,10 @@ These macros are obsolete, new ports should use the target hook
4467		4467
4468	@hook TARGET_MEMORY_MOVE_COST	4468	@hook TARGET_MEMORY_MOVE_COST
4469		4469
		4470	+@hook TARGET_CALLEE_SAVE_COST
		4471	+
		4472	+@hook TARGET_FRAME_ALLOCATION_COST
		4473	+
4470	@defmac BRANCH_COST (@var{speed_p}, @var{predictable_p})	4474	@defmac BRANCH_COST (@var{speed_p}, @var{predictable_p})
4471	A C expression for the cost of a branch instruction. A value of 1 is	4475	A C expression for the cost of a branch instruction. A value of 1 is
4472	the default; other values are interpreted relative to that. Parameter	4476	the default; other values are interpreted relative to that. Parameter

Mgcc/hard-reg-set.h+15-0

	@@ -191,6 +191,12 @@ hard_reg_set_empty_p (const_hard_reg_set x)
191	return x == HARD_CONST (0);	191	return x == HARD_CONST (0);
192	}	192	}
193		193
		194	+inline int
		195	+hard_reg_set_popcount (const_hard_reg_set x)
		196	+{
		197	+ return popcount_hwi (x);
		198	+}
		199	+
194	#else	200	#else
195		201
196	inline void	202	inline void
	@@ -254,6 +260,15 @@ hard_reg_set_empty_p (const_hard_reg_set x)
254	bad \|= x.elts[i];	260	bad \|= x.elts[i];
255	return bad == 0;	261	return bad == 0;
256	}	262	}
		263	+
		264	+inline int
		265	+hard_reg_set_popcount (const_hard_reg_set x)
		266	+{
		267	+ int count = 0;
		268	+ for (unsigned int i = 0; i < ARRAY_SIZE (x.elts); ++i)
		269	+ count += popcount_hwi (x.elts[i]);
		270	+ return count;
		271	+}
257	#endif	272	#endif
258		273
259	/* Iterator for hard register sets. */	274	/* Iterator for hard register sets. */

Mgcc/ira-color.cc+96-8

	@@ -1188,10 +1188,16 @@ finish_update_cost_records (void)
1188	update_cost_record_pool.release ();	1188	update_cost_record_pool.release ();
1189	}	1189	}
1190		1190
		1191	+/* True if we have allocated memory, or intend to do so. */
		1192	+static bool allocated_memory_p;
		1193	+
1191	/* Array whose element value is TRUE if the corresponding hard	1194	/* Array whose element value is TRUE if the corresponding hard
1192	register was already allocated for an allocno. */	1195	register was already allocated for an allocno. */
1193	static bool allocated_hardreg_p[FIRST_PSEUDO_REGISTER];	1196	static bool allocated_hardreg_p[FIRST_PSEUDO_REGISTER];
1194		1197
		1198	+/* Which callee-saved hard registers we've decided to save. */
		1199	+static HARD_REG_SET allocated_callee_save_regs;
		1200	+
1195	/* Describes one element in a queue of allocnos whose costs need to be	1201	/* Describes one element in a queue of allocnos whose costs need to be
1196	updated. Each allocno in the queue is known to have an allocno	1202	updated. Each allocno in the queue is known to have an allocno
1197	class. */	1203	class. */
	@@ -1733,6 +1739,20 @@ check_hard_reg_p (ira_allocno_t a, int hard_regno,
1733	return j == nregs;	1739	return j == nregs;
1734	}	1740	}
1735		1741
		1742	+/* Record that we have allocated NREGS registers starting at HARD_REGNO. */
		1743	+
		1744	+static void
		1745	+record_allocation (int hard_regno, int nregs)
		1746	+{
		1747	+ for (int i = 0; i < nregs; ++i)
		1748	+ if (!allocated_hardreg_p[hard_regno + i])
		1749	+ {
		1750	+ allocated_hardreg_p[hard_regno + i] = true;
		1751	+ if (!crtl->abi->clobbers_full_reg_p (hard_regno + i))
		1752	+ SET_HARD_REG_BIT (allocated_callee_save_regs, hard_regno + i);
		1753	+ }
		1754	+}
		1755	+
1736	/* Return number of registers needed to be saved and restored at	1756	/* Return number of registers needed to be saved and restored at
1737	function prologue/epilogue if we allocate HARD_REGNO to hold value	1757	function prologue/epilogue if we allocate HARD_REGNO to hold value
1738	of MODE. */	1758	of MODE. */
	@@ -1745,6 +1765,7 @@ calculate_saved_nregs (int hard_regno, machine_mode mode)
1745	ira_assert (hard_regno >= 0);	1765	ira_assert (hard_regno >= 0);
1746	for (i = hard_regno_nregs (hard_regno, mode) - 1; i >= 0; i--)	1766	for (i = hard_regno_nregs (hard_regno, mode) - 1; i >= 0; i--)
1747	if (!allocated_hardreg_p[hard_regno + i]	1767	if (!allocated_hardreg_p[hard_regno + i]
		1768	+ && ira_hard_regno_nrefs[hard_regno + i] == 0
1748	&& !crtl->abi->clobbers_full_reg_p (hard_regno + i)	1769	&& !crtl->abi->clobbers_full_reg_p (hard_regno + i)
1749	&& !LOCAL_REGNO (hard_regno + i))	1770	&& !LOCAL_REGNO (hard_regno + i))
1750	nregs++;	1771	nregs++;
	@@ -1953,6 +1974,12 @@ assign_hard_reg (ira_allocno_t a, bool retry_p)
1953	#endif	1974	#endif
1954	auto_bitmap allocnos_to_spill;	1975	auto_bitmap allocnos_to_spill;
1955	HARD_REG_SET soft_conflict_regs = {};	1976	HARD_REG_SET soft_conflict_regs = {};
		1977	+ int entry_freq = REG_FREQ_FROM_BB (ENTRY_BLOCK_PTR_FOR_FN (cfun));
		1978	+ int exit_freq = REG_FREQ_FROM_BB (EXIT_BLOCK_PTR_FOR_FN (cfun));
		1979	+ int spill_cost = 0;
		1980	+ /* Whether we have spilled pseudos or used caller-saved registers for values
		1981	+ that are live across a call. */
		1982	+ bool existing_spills_p = allocated_memory_p \|\| caller_save_needed;
1956		1983
1957	ira_assert (! ALLOCNO_ASSIGNED_P (a));	1984	ira_assert (! ALLOCNO_ASSIGNED_P (a));
1958	get_conflict_and_start_profitable_regs (a, retry_p,	1985	get_conflict_and_start_profitable_regs (a, retry_p,
	@@ -1971,7 +1998,19 @@ assign_hard_reg (ira_allocno_t a, bool retry_p)
1971	if (! retry_p)	1998	if (! retry_p)
1972	start_update_cost ();	1999	start_update_cost ();
1973	mem_cost += ALLOCNO_UPDATED_MEMORY_COST (a);	2000	mem_cost += ALLOCNO_UPDATED_MEMORY_COST (a);
1974	-	2001	+
		2002	+ if (!existing_spills_p)
		2003	+ {
		2004	+ auto entry_cost = targetm.frame_allocation_cost
		2005	+ (frame_cost_type::ALLOCATION, allocated_callee_save_regs);
		2006	+ spill_cost += entry_cost * entry_freq;
		2007	+
		2008	+ auto exit_cost = targetm.frame_allocation_cost
		2009	+ (frame_cost_type::DEALLOCATION, allocated_callee_save_regs);
		2010	+ spill_cost += exit_cost * exit_freq;
		2011	+ }
		2012	+ mem_cost += spill_cost;
		2013	+
1975	ira_allocate_and_copy_costs (&ALLOCNO_UPDATED_HARD_REG_COSTS (a),	2014	ira_allocate_and_copy_costs (&ALLOCNO_UPDATED_HARD_REG_COSTS (a),
1976	aclass, ALLOCNO_HARD_REG_COSTS (a));	2015	aclass, ALLOCNO_HARD_REG_COSTS (a));
1977	a_costs = ALLOCNO_UPDATED_HARD_REG_COSTS (a);	2016	a_costs = ALLOCNO_UPDATED_HARD_REG_COSTS (a);
	@@ -2165,15 +2204,37 @@ assign_hard_reg (ira_allocno_t a, bool retry_p)
2165	/* We need to save/restore the hard register in	2204	/* We need to save/restore the hard register in
2166	epilogue/prologue. Therefore we increase the cost. */	2205	epilogue/prologue. Therefore we increase the cost. */
2167	{	2206	{
		2207	+ int nregs = hard_regno_nregs (hard_regno, mode);
		2208	+ add_cost = 0;
2168	rclass = REGNO_REG_CLASS (hard_regno);	2209	rclass = REGNO_REG_CLASS (hard_regno);
2169	- add_cost = ((ira_memory_move_cost[mode][rclass][0]	2210	+
2170	- + ira_memory_move_cost[mode][rclass][1])	2211	+ auto entry_cost = targetm.callee_save_cost
2171	- * saved_nregs / hard_regno_nregs (hard_regno,	2212	+ (spill_cost_type::SAVE, hard_regno, mode, saved_nregs,
2172	- mode) - 1);	2213	+ ira_memory_move_cost[mode][rclass][0] * saved_nregs / nregs,
		2214	+ allocated_callee_save_regs, existing_spills_p);
		2215	+ /* In the event of a tie between caller-save and callee-save,
		2216	+ prefer callee-save. We apply this to the entry cost rather
		2217	+ than the exit cost since the entry frequency must be at
		2218	+ least as high as the exit frequency. */
		2219	+ if (entry_cost > 1)
		2220	+ entry_cost -= 1;
		2221	+ add_cost += entry_cost * entry_freq;
		2222	+
		2223	+ auto exit_cost = targetm.callee_save_cost
		2224	+ (spill_cost_type::RESTORE, hard_regno, mode, saved_nregs,
		2225	+ ira_memory_move_cost[mode][rclass][1] * saved_nregs / nregs,
		2226	+ allocated_callee_save_regs, existing_spills_p);
		2227	+ add_cost += exit_cost * exit_freq;
		2228	+
2173	cost += add_cost;	2229	cost += add_cost;
2174	full_cost += add_cost;	2230	full_cost += add_cost;
2175	}	2231	}
2176	}	2232	}
		2233	+ if (ira_need_caller_save_p (a, hard_regno))
		2234	+ {
		2235	+ cost += spill_cost;
		2236	+ full_cost += spill_cost;
		2237	+ }
2177	if (min_cost > cost)	2238	if (min_cost > cost)
2178	min_cost = cost;	2239	min_cost = cost;
2179	if (min_full_cost > full_cost)	2240	if (min_full_cost > full_cost)
	@@ -2200,11 +2261,13 @@ assign_hard_reg (ira_allocno_t a, bool retry_p)
2200	fail:	2261	fail:
2201	if (best_hard_regno >= 0)	2262	if (best_hard_regno >= 0)
2202	{	2263	{
2203	- for (i = hard_regno_nregs (best_hard_regno, mode) - 1; i >= 0; i--)	2264	+ record_allocation (best_hard_regno,
2204	- allocated_hardreg_p[best_hard_regno + i] = true;	2265	+ hard_regno_nregs (best_hard_regno, mode));
2205	spill_soft_conflicts (a, allocnos_to_spill, soft_conflict_regs,	2266	spill_soft_conflicts (a, allocnos_to_spill, soft_conflict_regs,
2206	best_hard_regno);	2267	best_hard_regno);
2207	}	2268	}
		2269	+ else
		2270	+ allocated_memory_p = true;
2208	if (! retry_p)	2271	if (! retry_p)
2209	restore_costs_from_copies (a);	2272	restore_costs_from_copies (a);
2210	ALLOCNO_HARD_REGNO (a) = best_hard_regno;	2273	ALLOCNO_HARD_REGNO (a) = best_hard_regno;
	@@ -3144,13 +3207,15 @@ improve_allocation (void)
3144	int j, k, n, hregno, conflict_hregno, base_cost, class_size, word, nwords;	3207	int j, k, n, hregno, conflict_hregno, base_cost, class_size, word, nwords;
3145	int check, spill_cost, min_cost, nregs, conflict_nregs, r, best;	3208	int check, spill_cost, min_cost, nregs, conflict_nregs, r, best;
3146	bool try_p;	3209	bool try_p;
3147	- enum reg_class aclass;	3210	+ enum reg_class aclass, rclass;
3148	machine_mode mode;	3211	machine_mode mode;
3149	int *allocno_costs;	3212	int *allocno_costs;
3150	int costs[FIRST_PSEUDO_REGISTER];	3213	int costs[FIRST_PSEUDO_REGISTER];
3151	HARD_REG_SET conflicting_regs[2], profitable_hard_regs;	3214	HARD_REG_SET conflicting_regs[2], profitable_hard_regs;
3152	ira_allocno_t a;	3215	ira_allocno_t a;
3153	bitmap_iterator bi;	3216	bitmap_iterator bi;
		3217	+ int saved_nregs;
		3218	+ int add_cost;
3154		3219
3155	/* Don't bother to optimize the code with static chain pointer and	3220	/* Don't bother to optimize the code with static chain pointer and
3156	non-local goto in order not to spill the chain pointer	3221	non-local goto in order not to spill the chain pointer
	@@ -3188,6 +3253,7 @@ improve_allocation (void)
3188	conflicting_regs,	3253	conflicting_regs,
3189	&profitable_hard_regs);	3254	&profitable_hard_regs);
3190	class_size = ira_class_hard_regs_num[aclass];	3255	class_size = ira_class_hard_regs_num[aclass];
		3256	+ mode = ALLOCNO_MODE (a);
3191	/* Set up cost improvement for usage of each profitable hard	3257	/* Set up cost improvement for usage of each profitable hard
3192	register for allocno A. */	3258	register for allocno A. */
3193	for (j = 0; j < class_size; j++)	3259	for (j = 0; j < class_size; j++)
	@@ -3201,6 +3267,22 @@ improve_allocation (void)
3201	costs[hregno] = (allocno_costs == NULL	3267	costs[hregno] = (allocno_costs == NULL
3202	? ALLOCNO_UPDATED_CLASS_COST (a) : allocno_costs[k]);	3268	? ALLOCNO_UPDATED_CLASS_COST (a) : allocno_costs[k]);
3203	costs[hregno] -= allocno_copy_cost_saving (a, hregno);	3269	costs[hregno] -= allocno_copy_cost_saving (a, hregno);
		3270	+
		3271	+ if ((saved_nregs = calculate_saved_nregs (hregno, mode)) != 0)
		3272	+ {
		3273	+ /* We need to save/restore the hard register in
		3274	+ epilogue/prologue. Therefore we increase the cost.
		3275	+ Since the prolog is placed in the entry BB, the frequency
		3276	+ of the entry BB is considered while computing the cost. */
		3277	+ rclass = REGNO_REG_CLASS (hregno);
		3278	+ add_cost = ((ira_memory_move_cost[mode][rclass][0]
		3279	+ + ira_memory_move_cost[mode][rclass][1])
		3280	+ * saved_nregs / hard_regno_nregs (hregno,
		3281	+ mode) - 1)
		3282	+ * REG_FREQ_FROM_BB (ENTRY_BLOCK_PTR_FOR_FN (cfun));
		3283	+ costs[hregno] += add_cost;
		3284	+ }
		3285	+
3204	costs[hregno] -= base_cost;	3286	costs[hregno] -= base_cost;
3205	if (costs[hregno] < 0)	3287	if (costs[hregno] < 0)
3206	try_p = true;	3288	try_p = true;
	@@ -3334,6 +3416,7 @@ improve_allocation (void)
3334	}	3416	}
3335	/* Assign the best chosen hard register to A. */	3417	/* Assign the best chosen hard register to A. */
3336	ALLOCNO_HARD_REGNO (a) = best;	3418	ALLOCNO_HARD_REGNO (a) = best;
		3419	+ record_allocation (best, nregs);
3337	if (internal_flag_ira_verbose > 2 && ira_dump_file != NULL)	3420	if (internal_flag_ira_verbose > 2 && ira_dump_file != NULL)
3338	fprintf (ira_dump_file, "Assigning %d to a%dr%d\n",	3421	fprintf (ira_dump_file, "Assigning %d to a%dr%d\n",
3339	best, ALLOCNO_NUM (a), ALLOCNO_REGNO (a));	3422	best, ALLOCNO_NUM (a), ALLOCNO_REGNO (a));
	@@ -5162,6 +5245,7 @@ color (void)
5162	{	5245	{
5163	allocno_stack_vec.create (ira_allocnos_num);	5246	allocno_stack_vec.create (ira_allocnos_num);
5164	memset (allocated_hardreg_p, 0, sizeof (allocated_hardreg_p));	5247	memset (allocated_hardreg_p, 0, sizeof (allocated_hardreg_p));
		5248	+ CLEAR_HARD_REG_SET (allocated_callee_save_regs);
5165	ira_initiate_assign ();	5249	ira_initiate_assign ();
5166	do_coloring ();	5250	do_coloring ();
5167	ira_finish_assign ();	5251	ira_finish_assign ();
	@@ -5286,10 +5370,14 @@ ira_color (void)
5286	ira_allocno_iterator ai;	5370	ira_allocno_iterator ai;
5287		5371
5288	/* Setup updated costs. */	5372	/* Setup updated costs. */
		5373	+ allocated_memory_p = false;
5289	FOR_EACH_ALLOCNO (a, ai)	5374	FOR_EACH_ALLOCNO (a, ai)
5290	{	5375	{
5291	ALLOCNO_UPDATED_MEMORY_COST (a) = ALLOCNO_MEMORY_COST (a);	5376	ALLOCNO_UPDATED_MEMORY_COST (a) = ALLOCNO_MEMORY_COST (a);
5292	ALLOCNO_UPDATED_CLASS_COST (a) = ALLOCNO_CLASS_COST (a);	5377	ALLOCNO_UPDATED_CLASS_COST (a) = ALLOCNO_CLASS_COST (a);
		5378	+ if (ALLOCNO_CLASS (a) == NO_REGS
		5379	+ && !ira_equiv_no_lvalue_p (ALLOCNO_REGNO (a)))
		5380	+ allocated_memory_p = true;
5293	}	5381	}
5294	if (ira_conflicts_p)	5382	if (ira_conflicts_p)
5295	color ();	5383	color ();

Mgcc/ira-int.h+5-0

	@@ -922,6 +922,9 @@ public:
922		922
923	/* Flag of that the above array has been initialized. */	923	/* Flag of that the above array has been initialized. */
924	bool x_ira_prohibited_mode_move_regs_initialized_p;	924	bool x_ira_prohibited_mode_move_regs_initialized_p;
		925	+
		926	+ /* Number of real occurrences of hard regs before IRA. */
		927	+ size_t x_ira_hard_regno_nrefs[FIRST_PSEUDO_REGISTER];
925	};	928	};
926		929
927	extern class target_ira_int default_target_ira_int;	930	extern class target_ira_int default_target_ira_int;
	@@ -969,6 +972,8 @@ extern class target_ira_int *this_target_ira_int;
969	(this_target_ira_int->x_ira_reg_class_superunion)	972	(this_target_ira_int->x_ira_reg_class_superunion)
970	#define ira_prohibited_mode_move_regs \	973	#define ira_prohibited_mode_move_regs \
971	(this_target_ira_int->x_ira_prohibited_mode_move_regs)	974	(this_target_ira_int->x_ira_prohibited_mode_move_regs)
		975	+#define ira_hard_regno_nrefs \
		976	+ (this_target_ira_int->x_ira_hard_regno_nrefs)
972		977
973	/* ira.cc: */	978	/* ira.cc: */
974		979

Mgcc/ira.cc+31-23

	@@ -1409,34 +1409,17 @@ find_reg_classes (void)
1409		1409
1410		1410
1411		1411
1412	-/* Set up the array above. */	1412	+/* Set up array ira_hard_regno_allocno_class. */
1413	static void	1413	static void
1414	setup_hard_regno_aclass (void)	1414	setup_hard_regno_aclass (void)
1415	{	1415	{
1416	int i;	1416	int i;
1417		1417
1418	for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)	1418	for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
1419	- {	1419	+ ira_hard_regno_allocno_class[i]
1420	-#if 1	1420	+ = (TEST_HARD_REG_BIT (no_unit_alloc_regs, i)
1421	- ira_hard_regno_allocno_class[i]	1421	+ ? NO_REGS
1422	- = (TEST_HARD_REG_BIT (no_unit_alloc_regs, i)	1422	+ : ira_allocno_class_translate[REGNO_REG_CLASS (i)]);
1423	- ? NO_REGS
1424	- : ira_allocno_class_translate[REGNO_REG_CLASS (i)]);
1425	-#else
1426	- int j;
1427	- enum reg_class cl;
1428	- ira_hard_regno_allocno_class[i] = NO_REGS;
1429	- for (j = 0; j < ira_allocno_classes_num; j++)
1430	- {
1431	- cl = ira_allocno_classes[j];
1432	- if (ira_class_hard_reg_index[cl][i] >= 0)
1433	- {
1434	- ira_hard_regno_allocno_class[i] = cl;
1435	- break;
1436	- }
1437	- }
1438	-#endif
1439	- }
1440	}	1423	}
1441		1424
1442		1425
	@@ -5517,6 +5500,30 @@ bool ira_conflicts_p;
5517	/* Saved between IRA and reload. */	5500	/* Saved between IRA and reload. */
5518	static int saved_flag_ira_share_spill_slots;	5501	static int saved_flag_ira_share_spill_slots;
5519		5502
		5503	+/* Set up array ira_hard_regno_nrefs. */
		5504	+static void
		5505	+setup_hard_regno_nrefs (void)
		5506	+{
		5507	+ int i;
		5508	+
		5509	+ for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
		5510	+ {
		5511	+ ira_hard_regno_nrefs[i] = 0;
		5512	+ for (df_ref use = DF_REG_USE_CHAIN (i);
		5513	+ use != NULL;
		5514	+ use = DF_REF_NEXT_REG (use))
		5515	+ if (DF_REF_CLASS (use) != DF_REF_ARTIFICIAL
		5516	+ && !(DF_REF_INSN_INFO (use) && DEBUG_INSN_P (DF_REF_INSN (use))))
		5517	+ ira_hard_regno_nrefs[i]++;
		5518	+ for (df_ref def = DF_REG_DEF_CHAIN (i);
		5519	+ def != NULL;
		5520	+ def = DF_REF_NEXT_REG (def))
		5521	+ if (DF_REF_CLASS (def) != DF_REF_ARTIFICIAL
		5522	+ && !(DF_REF_INSN_INFO (def) && DEBUG_INSN_P (DF_REF_INSN (def))))
		5523	+ ira_hard_regno_nrefs[i]++;
		5524	+ }
		5525	+}
		5526	+
5520	/* This is the main entry of IRA. */	5527	/* This is the main entry of IRA. */
5521	static void	5528	static void
5522	ira (FILE *f)	5529	ira (FILE *f)
	@@ -5529,7 +5536,8 @@ ira (FILE *f)
5529	edge_iterator ei;	5536	edge_iterator ei;
5530	edge e;	5537	edge e;
5531	bool output_jump_reload_p = false;	5538	bool output_jump_reload_p = false;
5532	-	5539	+
		5540	+ setup_hard_regno_nrefs ();
5533	if (ira_use_lra_p)	5541	if (ira_use_lra_p)
5534	{	5542	{
5535	/* First put potential jump output reloads on the output edges	5543	/* First put potential jump output reloads on the output edges

Mgcc/target.def+75-1

	@@ -3758,6 +3758,81 @@ are the same as to this target hook.",
3758	int, (machine_mode mode, reg_class_t rclass, bool in),	3758	int, (machine_mode mode, reg_class_t rclass, bool in),
3759	default_memory_move_cost)	3759	default_memory_move_cost)
3760		3760
		3761	+DEFHOOK
		3762	+(callee_save_cost,
		3763	+ "Return the one-off cost of saving or restoring callee-saved registers\n\
		3764	+(also known as call-preserved registers or non-volatile registers).\n\
		3765	+The parameters are as follows:\n\
		3766	+\n\
		3767	+@itemize\n\
		3768	+@item\n\
		3769	+@var{cost_type} is @samp{spill_cost_type::SAVE} for saving a register\n\
		3770	+and @samp{spill_cost_type::RESTORE} for restoring a register.\n\
		3771	+\n\
		3772	+@item\n\
		3773	+@var{hard_regno} and @var{mode} represent the whole register that\n\
		3774	+the register allocator is considering using; of these,\n\
		3775	+@var{nregs} registers are fully or partially callee-saved.\n\
		3776	+\n\
		3777	+@item\n\
		3778	+@var{mem_cost} is the normal cost for storing (for saves)\n\
		3779	+or loading (for restores) the @var{nregs} registers.\n\
		3780	+\n\
		3781	+@item\n\
		3782	+@var{allocated_callee_regs} is the set of callee-saved registers\n\
		3783	+that are already in use.\n\
		3784	+\n\
		3785	+@item\n\
		3786	+@var{existing_spills_p} is true if the register allocator has\n\
		3787	+already decided to spill registers to memory.\n\
		3788	+@end itemize\n\
		3789	+\n\
		3790	+If @var{existing_spills_p} is false, the cost of a save should account\n\
		3791	+for frame allocations in a way that is consistent with\n\
		3792	+@code{TARGET_FRAME_ALLOCATION_COST}'s handling of allocations for spills.\n\
		3793	+Similarly, the cost of a restore should then account for frame deallocations\n\
		3794	+in a way that is consistent with @code{TARGET_FRAME_ALLOCATION_COST}'s\n\
		3795	+handling of deallocations.\n\
		3796	+\n\
		3797	+Note that this hook should not attempt to apply a frequency scale\n\
		3798	+to the cost: it is the caller's responsibility to do that where\n\
		3799	+appropriate.\n\
		3800	+\n\
		3801	+The default implementation returns @var{mem_cost}, plus the allocation\n\
		3802	+or deallocation cost returned by @code{TARGET_FRAME_ALLOCATION_COST},\n\
		3803	+where appropriate.",
		3804	+ int, (spill_cost_type cost_type, unsigned int hard_regno,
		3805	+ machine_mode mode, unsigned int nregs, int mem_cost,
		3806	+ const HARD_REG_SET &allocated_callee_regs, bool existing_spills_p),
		3807	+ default_callee_save_cost)
		3808	+
		3809	+DEFHOOK
		3810	+(frame_allocation_cost,
		3811	+ "Return the cost of allocating or deallocating a frame for the sake of\n\
		3812	+a spill; @var{cost_type} chooses between allocation and deallocation.\n\
		3813	+The term ``spill'' here includes both forcing a pseudo register to memory\n\
		3814	+and using caller-saved registers for pseudo registers that are live across\n\
		3815	+a call.\n\
		3816	+\n\
		3817	+This hook is only called if the register allocator has not so far\n\
		3818	+decided to spill. The allocator may have decided to use callee-saved\n\
		3819	+registers; if so, @var{allocated_callee_regs} is the set of callee-saved\n\
		3820	+registers that the allocator has used. There might also be other reasons\n\
		3821	+why a stack frame is already needed; for example, @samp{get_frame_size ()}\n\
		3822	+might be nonzero, or the target might already require a frame for\n\
		3823	+target-specific reasons.\n\
		3824	+\n\
		3825	+When the register allocator uses this hook to cost spills, it also uses\n\
		3826	+@code{TARGET_CALLEE_SAVE_COST} to cost new callee-saved registers, passing\n\
		3827	+@samp{false} as the @var{existing_spills_p} argument. The intention is to\n\
		3828	+allow the target to apply an apples-for-apples comparison between the\n\
		3829	+cost of using callee-saved registers and using spills in cases where the\n\
		3830	+allocator has not yet committed to using both strategies.\n\
		3831	+\n\
		3832	+The default implementation returns 0.",
		3833	+ int, (frame_cost_type cost_type, const HARD_REG_SET &allocated_callee_regs),
		3834	+ default_frame_allocation_cost)
		3835	+
3761	DEFHOOK	3836	DEFHOOK
3762	(use_by_pieces_infrastructure_p,	3837	(use_by_pieces_infrastructure_p,
3763	"GCC will attempt several strategies when asked to copy between\n\	3838	"GCC will attempt several strategies when asked to copy between\n\
	@@ -7379,4 +7454,3 @@ DEFHOOKPOD
7379		7454
7380	/* Close the 'struct gcc_target' definition. */	7455	/* Close the 'struct gcc_target' definition. */
7381	HOOK_VECTOR_END (C90_EMPTY_HACK)	7456	HOOK_VECTOR_END (C90_EMPTY_HACK)
7382	-

Mgcc/target.h+12-0

	@@ -259,6 +259,18 @@ enum poly_value_estimate_kind
259	POLY_VALUE_LIKELY	259	POLY_VALUE_LIKELY
260	};	260	};
261		261
		262	+enum class spill_cost_type
		263	+{
		264	+ SAVE,
		265	+ RESTORE
		266	+};
		267	+
		268	+enum class frame_cost_type
		269	+{
		270	+ ALLOCATION,
		271	+ DEALLOCATION
		272	+};
		273	+
262	extern bool verify_type_context (location_t, type_context_kind, const_tree,	274	extern bool verify_type_context (location_t, type_context_kind, const_tree,
263	bool = false);	275	bool = false);
264		276

Mgcc/targhooks.cc+27-0

	@@ -1903,6 +1903,33 @@ default_register_move_cost (machine_mode mode ATTRIBUTE_UNUSED,
1903	#endif	1903	#endif
1904	}	1904	}
1905		1905
		1906	+/* The default implementation of TARGET_CALLEE_SAVE_COST. */
		1907	+
		1908	+int
		1909	+default_callee_save_cost (spill_cost_type spill_type, unsigned int,
		1910	+ machine_mode, unsigned int, int mem_cost,
		1911	+ const HARD_REG_SET &callee_saved_regs,
		1912	+ bool existing_spills_p)
		1913	+{
		1914	+ if (!existing_spills_p)
		1915	+ {
		1916	+ auto frame_type = (spill_type == spill_cost_type::SAVE
		1917	+ ? frame_cost_type::ALLOCATION
		1918	+ : frame_cost_type::DEALLOCATION);
		1919	+ mem_cost += targetm.frame_allocation_cost (frame_type,
		1920	+ callee_saved_regs);
		1921	+ }
		1922	+ return mem_cost;
		1923	+}
		1924	+
		1925	+/* The default implementation of TARGET_FRAME_ALLOCATION_COST. */
		1926	+
		1927	+int
		1928	+default_frame_allocation_cost (frame_cost_type, const HARD_REG_SET &)
		1929	+{
		1930	+ return 0;
		1931	+}
		1932	+
1906	/* The default implementation of TARGET_SLOW_UNALIGNED_ACCESS. */	1933	/* The default implementation of TARGET_SLOW_UNALIGNED_ACCESS. */
1907		1934
1908	bool	1935	bool

Mgcc/targhooks.h+5-0

	@@ -220,6 +220,11 @@ extern tree default_builtin_tm_load_store (tree);
220	extern int default_memory_move_cost (machine_mode, reg_class_t, bool);	220	extern int default_memory_move_cost (machine_mode, reg_class_t, bool);
221	extern int default_register_move_cost (machine_mode, reg_class_t,	221	extern int default_register_move_cost (machine_mode, reg_class_t,
222	reg_class_t);	222	reg_class_t);
		223	+extern int default_callee_save_cost (spill_cost_type, unsigned int,
		224	+ machine_mode, unsigned int, int,
		225	+ const HARD_REG_SET &, bool);
		226	+extern int default_frame_allocation_cost (frame_cost_type,
		227	+ const HARD_REG_SET &);
223	extern bool default_slow_unaligned_access (machine_mode, unsigned int);	228	extern bool default_slow_unaligned_access (machine_mode, unsigned int);
224	extern HOST_WIDE_INT default_estimated_poly_value (poly_int64,	229	extern HOST_WIDE_INT default_estimated_poly_value (poly_int64,
225	poly_value_estimate_kind);	230	poly_value_estimate_kind);

Agcc/testsuite/gcc.dg/rtl/aarch64/pr122827.c+33-0

@@ -0,0 +1,33 @@
	1	+/* { dg-do compile { target aarch64-- } } */
	2	+/* { dg-options "-O2 -fno-caller-saves -march=armv8.2-a+sve -msve-vector-bits=128" } */
	3	+
	4	+/* A VNx2SF value occupies a full Z register even though its data size is
	5	+ only 64 bits for -msve-vector-bits=128. It therefore cannot be kept in
	6	+ V8-V15 across an ordinary call, since AAPCS64 only preserves D8-D15. */
	7	+
	8	+void __RTL (startwith ("ira")) test (void)
	9	+{
	10	+(function "test"
	11	+ (insn-chain
	12	+ (block 2
	13	+ (edge-from entry (flags "FALLTHRU"))
	14	+ (cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
	15	+ (insn 2 (set (reg:VNx2SF <0>) (reg:VNx2SF v0)))
	16	+ (call_insn 3 (parallel [
	17	+ (call (mem:DI (symbol_ref:DI ("clobber") [flags 0x41])
	18	+ [0 clobber S8 A8])
	19	+ (const_int 0))
	20	+ (unspec:DI [(const_int 2)] UNSPEC_CALLEE_ABI)
	21	+ (clobber (reg:DI x30))
	22	+ ]))
	23	+ (insn 4 (set (reg:VNx2SF v0) (reg:VNx2SF <0>)))
	24	+ (insn 5 (use (reg:VNx2SF v0)))
	25	+ (edge-to exit (flags "FALLTHRU"))
	26	+ )
	27	+ )
	28	+)
	29	+}
	30	+
	31	+/* Before PR target/122827, this sequence used a Z8-Z15 register across the
	32	+ call while saving and restoring only its low 64 bits. */
	33	+/* { dg-final { scan-assembler-not {mov\tz(8\|9\|1[0-5])\.d,} } } */

Agcc/testsuite/gcc.target/aarch64/callee_save_1.c+12-0

@@ -0,0 +1,12 @@
	1	+/* { dg-options "-O2" } */
	2	+
	3	+int test (int x), test2 (int x);
	4	+
	5	+int foo (int x, int y) {
	6	+ test (x);
	7	+ int lhs = test2 (y);
	8	+ return x + lhs;
	9	+}
	10	+
	11	+/* { dg-final { scan-assembler {\tstp\tx19, x20, \[sp,} } } */
	12	+/* { dg-final { scan-assembler {\tldp\tx19, x20, \[sp,} } } */

Agcc/testsuite/gcc.target/aarch64/callee_save_2.c+16-0

@@ -0,0 +1,16 @@
	1	+/* { dg-options "-O2 -fomit-frame-pointer" } */
	2	+
	3	+int test (int x), test2 (int x);
	4	+
	5	+int foo (int x, int y) {
	6	+ test (x);
	7	+ int lhs = test2 (y);
	8	+ return x + lhs;
	9	+}
	10	+
	11	+/* { dg-final { scan-assembler {\tstp\tx30, x19, \[sp,} } } */
	12	+/* { dg-final { scan-assembler {\tldp\tx30, x19, \[sp\],} } } */
	13	+/* GCC 12 expands incoming arguments through additional pseudos, so the
	14	+ second value does not retain a profitable preference for x1. */
	15	+/* { dg-final { scan-assembler {\tstr\tx20, \[sp,} } } */
	16	+/* { dg-final { scan-assembler {\tldr\tx20, \[sp,} } } */

Agcc/testsuite/gcc.target/aarch64/callee_save_3.c+13-0

@@ -0,0 +1,13 @@
	1	+/* { dg-options "-O2" } */
	2	+
	3	+float test ();
	4	+float g;
	5	+
	6	+float foo (float x, float y) {
	7	+ g = x + test ();
	8	+ return (x + test ()) * y;
	9	+}
	10	+
	11	+/* GCC 12 allocates FP registers in ascending register-number order. */
	12	+/* { dg-final { scan-assembler {\tstp\td8, d9, \[sp,} } } */
	13	+/* { dg-final { scan-assembler {\tldp\td8, d9, \[sp,} } } */

Mgcc/testsuite/gcc.target/aarch64/pr103350-1.c+1-1

	@@ -1,5 +1,5 @@
1	/* { dg-do run { target le } } */	1	/* { dg-do run { target le } } */
2	-/* { dg-additional-options "-Os -fno-tree-ter -save-temps -fdump-rtl-ree-all -free -std=c99 -w" } */	2	+/* { dg-additional-options "-Os -fno-tree-ter -save-temps -fdump-rtl-ree-all -free -std=c99 -w -fno-caller-saves" } */
3		3
4	typedef unsigned char u8;	4	typedef unsigned char u8;
5	typedef unsigned char __attribute__((__vector_size__ (8))) v64u8;	5	typedef unsigned char __attribute__((__vector_size__ (8))) v64u8;

Agcc/testsuite/gcc.target/aarch64/sve/pr122827.c+95-0

@@ -0,0 +1,95 @@
	1	+/* PR target/122827.
	2	+
	3	+ With -msve-vector-bits=128 and SVE-only autovec, the inlined
	4	+ madd_v3 calls are vectorised into SVE partial-mode FMLAs that share
	5	+ a single vec_duplicate of the scalar FAC.
	6	+ Before the fix, the register allocator placed the VNx2SF
	7	+ broadcast in V14 across the call to clobber() because IRA thought
	8	+ V14 was preserved (only the low 64 bits, D14, are callee-saved under
	9	+ AAPCS64). The SVE FMLA reads the full Z register, so the upper
	10	+ lanes of the broadcast are corrupted by the call and the result of
	11	+ the FMLA is wrong. */
	12	+
	13	+/* { dg-do run { target aarch64_sve128_hw } } */
	14	+/* { dg-options "-Ofast -msve-vector-bits=128 --param=aarch64-autovec-preference=2" } */
	15	+/* { dg-require-effective-target aarch64_little_endian } */
	16	+
	17	+struct R
	18	+{
	19	+ float combined[3];
	20	+ float spec[3];
	21	+ float diff[3];
	22	+ float diffshad[3];
	23	+ float shad[3];
	24	+};
	25	+
	26	+struct I
	27	+{
	28	+ float fac;
	29	+ int passflag;
	30	+};
	31	+
	32	+/* AAPCS64 only requires callees to preserve the low 64 bits of V8-V15;
	33	+ the upper lanes are caller-clobbered. The buggy register allocator
	34	+ in this test places the broadcast in V14 specifically, so clobbering
	35	+ V14s upper bits here is sufficient to expose the miscompile. The
	36	+ asm clobber list lets GCC save/restore D14 around the write, so we
	37	+ honour the ABI contract for the low half while leaving the upper
	38	+ half trashed, what any normal callee is permitted to do. */
	39	+__attribute__((noipa)) static void
	40	+clobber (struct I si, struct R r)
	41	+{
	42	+ asm volatile ("movi v14.16b, #0xff" : : : "v14");
	43	+}
	44	+
	45	+static inline __attribute__((always_inline)) void
	46	+madd_v3 (float r[3], const float a[3], float f)
	47	+{
	48	+ r[0] += a[0] * f;
	49	+ r[1] += a[1] * f;
	50	+ r[2] += a[2] * f;
	51	+}
	52	+
	53	+__attribute__((noipa)) void
	54	+test (struct I si, struct R r)
	55	+{
	56	+ struct R t = { { 1, 2, 3 }, { 4, 5, 6 }, { 7, 8, 9 },
	57	+ { 10, 11, 12 }, { 13, 14, 15 } };
	58	+
	59	+ clobber (si, r);
	60	+ if (si->fac != 0.0f)
	61	+ {
	62	+ float fac = si->fac;
	63	+ clobber (si, &t);
	64	+ madd_v3 (r->combined, t.combined, fac);
	65	+ if (si->passflag & 1)
	66	+ madd_v3 (r->spec, t.spec, fac);
	67	+ if (si->passflag & 2)
	68	+ {
	69	+ madd_v3 (r->diff, t.diff, fac);
	70	+ madd_v3 (r->diffshad, t.diffshad, fac);
	71	+ }
	72	+ if (si->passflag & 4)
	73	+ madd_v3 (r->shad, t.shad, fac);
	74	+ }
	75	+}
	76	+
	77	+int
	78	+main (void)
	79	+{
	80	+ struct I si = { 2.0f, 7 };
	81	+ struct R r = { { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 },
	82	+ { 0, 0, 0 }, { 0, 0, 0 } };
	83	+ test (&si, &r);
	84	+ if (r.combined[0] != 2.f \|\| r.combined[1] != 4.f \|\| r.combined[2] != 6.f)
	85	+ __builtin_abort ();
	86	+ if (r.spec[0] != 8.f \|\| r.spec[1] != 10.f \|\| r.spec[2] != 12.f)
	87	+ __builtin_abort ();
	88	+ if (r.diff[0] != 14.f \|\| r.diff[1] != 16.f \|\| r.diff[2] != 18.f)
	89	+ __builtin_abort ();
	90	+ if (r.diffshad[0] != 20.f \|\| r.diffshad[1] != 22.f \|\| r.diffshad[2] != 24.f)
	91	+ __builtin_abort ();
	92	+ if (r.shad[0] != 26.f \|\| r.shad[1] != 28.f \|\| r.shad[2] != 30.f)
	93	+ __builtin_abort ();
	94	+ return 0;
	95	+}

Mgcc/testsuite/gcc.target/i386/force-indirect-call-2.c+1-1

	@@ -1,6 +1,6 @@
1	/* { dg-do compile } */	1	/* { dg-do compile } */
2	/* { dg-options "-O2 -mforce-indirect-call -fPIC" } */	2	/* { dg-options "-O2 -mforce-indirect-call -fPIC" } */
3	/* { dg-require-effective-target fpic } */	3	/* { dg-require-effective-target fpic } */
4	-/* { dg-final { scan-assembler-times "(?:call\|jmp)\[ \\t\]+\\%" 3 } } /	4	+/* { dg-final { scan-assembler-times "(?:call\|jmp)\[ \\t\]+\\" 3 } } /
5		5
6	#include "force-indirect-call-1.c"	6	#include "force-indirect-call-1.c"

Agcc/testsuite/gcc.target/i386/pr110071.c+16-0

@@ -0,0 +1,16 @@
	1	+/* { dg-do compile { target { ! ia32 } } } */
	2	+/* { dg-options "-O2 -fdump-rtl-pro_and_epilogue" } */
	3	+
	4	+/* Verify there is an early return without the prolog and shrink-wrap
	5	+ the function. */
	6	+
	7	+void bar ();
	8	+long
	9	+foo (long i, long cond)
	10	+{
	11	+ if (cond)
	12	+ bar ();
	13	+ return i+1;
	14	+}
	15	+
	16	+/* { dg-final { scan-rtl-dump-times "Performing shrink-wrapping" 1 "pro_and_epilogue" } } */

Agcc/testsuite/gcc.target/i386/pr111673.c+17-0

@@ -0,0 +1,17 @@
	1	+/* { dg-do compile { target { ! ia32 } } } */
	2	+/* { dg-options "-O2 -fdump-rtl-pro_and_epilogue" } */
	3	+
	4	+/* Verify there is an early return without the prolog and shrink-wrap
	5	+ the function. */
	6	+
	7	+int f (int);
	8	+int
	9	+advance (int dz)
	10	+{
	11	+ if (dz > 0)
	12	+ return (dz + dz) * dz;
	13	+ else
	14	+ return dz * f (dz);
	15	+}
	16	+
	17	+/* { dg-final { scan-rtl-dump-times "Performing shrink-wrapping" 1 "pro_and_epilogue" } } */

Agcc/testsuite/gcc.target/i386/pr118497.c+17-0

@@ -0,0 +1,17 @@
	1	+/* { dg-do compile { target { ia32 } } } */
	2	+/* { dg-options "-O2 -fpic" } */
	3	+
	4	+extern void crosscall2 (void (fn) (void , int), void *, int);
	5	+extern void _cgo_panic (void *, int);
	6	+extern void _cgo_allocate (void *, int);
	7	+
	8	+void
	9	+callPanic (void)
	10	+{
	11	+ struct { const char *p; } a;
	12	+ a.p = "panic from C";
	13	+ crosscall2 (_cgo_panic, &a, sizeof a);
	14	+ (int ) 1 = 1;
	15	+}
	16	+
	17	+/* { dg-final { scan-assembler "__x86.get_pc_thunk.bx" } } */

Agcc/testsuite/gcc.target/powerpc/pr110071.c+15-0

@@ -0,0 +1,15 @@
	1	+/* { dg-do compile { target lp64 } } */
	2	+/* { dg-options "-O2 -fdump-rtl-pro_and_epilogue" } */
	3	+
	4	+/* Verify there is an early return without the prolog and shrink-wrap
	5	+ the function. */
	6	+void bar ();
	7	+long
	8	+foo (long i, long cond)
	9	+{
	10	+ if (cond)
	11	+ bar ();
	12	+ return i+1;
	13	+}
	14	+
	15	+/* { dg-final { scan-rtl-dump-times "Performing shrink-wrapping" 1 "pro_and_epilogue" } } */

Agcc/testsuite/gcc.target/powerpc/pr111673.c+17-0

@@ -0,0 +1,17 @@
	1	+/* { dg-do compile { target lp64 } } */
	2	+/* { dg-options "-O2 -fdump-rtl-pro_and_epilogue" } */
	3	+
	4	+/* Verify there is an early return without the prolog and shrink-wrap
	5	+ the function. */
	6	+
	7	+int f (int);
	8	+int
	9	+advance (int dz)
	10	+{
	11	+ if (dz > 0)
	12	+ return (dz + dz) * dz;
	13	+ else
	14	+ return dz * f (dz);
	15	+}
	16	+
	17	+/* { dg-final { scan-rtl-dump-times "Performing shrink-wrapping" 1 "pro_and_epilogue" } } */