@@ -818,81 +818,239 @@ public:
#undef INSN
-enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11};
-
-#define INSN(NAME, op, funct3, funct7) \
- void NAME(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) { \
- unsigned insn = 0; \
- patch((address)&insn, 6, 0, op); \
- patch((address)&insn, 14, 12, funct3); \
- patch_reg((address)&insn, 7, Rd); \
- patch_reg((address)&insn, 15, Rs1); \
- patch_reg((address)&insn, 20, Rs2); \
- patch((address)&insn, 31, 27, funct7); \
- patch((address)&insn, 26, 25, memory_order); \
- emit(insn); \
- }
-
- INSN(amoswap_w, 0b0101111, 0b010, 0b00001);
- INSN(amoadd_w, 0b0101111, 0b010, 0b00000);
- INSN(amoxor_w, 0b0101111, 0b010, 0b00100);
- INSN(amoand_w, 0b0101111, 0b010, 0b01100);
- INSN(amoor_w, 0b0101111, 0b010, 0b01000);
- INSN(amomin_w, 0b0101111, 0b010, 0b10000);
- INSN(amomax_w, 0b0101111, 0b010, 0b10100);
- INSN(amominu_w, 0b0101111, 0b010, 0b11000);
- INSN(amomaxu_w, 0b0101111, 0b010, 0b11100);
- INSN(amoswap_d, 0b0101111, 0b011, 0b00001);
- INSN(amoadd_d, 0b0101111, 0b011, 0b00000);
- INSN(amoxor_d, 0b0101111, 0b011, 0b00100);
- INSN(amoand_d, 0b0101111, 0b011, 0b01100);
- INSN(amoor_d, 0b0101111, 0b011, 0b01000);
- INSN(amomin_d, 0b0101111, 0b011, 0b10000);
- INSN(amomax_d , 0b0101111, 0b011, 0b10100);
- INSN(amominu_d, 0b0101111, 0b011, 0b11000);
- INSN(amomaxu_d, 0b0101111, 0b011, 0b11100);
- INSN(amocas_w, 0b0101111, 0b010, 0b00101);
- INSN(amocas_d, 0b0101111, 0b011, 0b00101);
-#undef INSN
-
-enum operand_size { int8, int16, int32, uint32, int64 };
-
-#define INSN(NAME, op, funct3, funct7) \
- void NAME(Register Rd, Register Rs1, Aqrl memory_order = relaxed) { \
- unsigned insn = 0; \
- uint32_t val = memory_order & 0x3; \
- patch((address)&insn, 6, 0, op); \
- patch((address)&insn, 14, 12, funct3); \
- patch_reg((address)&insn, 7, Rd); \
- patch_reg((address)&insn, 15, Rs1); \
- patch((address)&insn, 25, 20, 0b00000); \
- patch((address)&insn, 31, 27, funct7); \
- patch((address)&insn, 26, 25, val); \
- emit(insn); \
- }
-
- INSN(lr_w, 0b0101111, 0b010, 0b00010);
- INSN(lr_d, 0b0101111, 0b011, 0b00010);
-
-#undef INSN
-
-#define INSN(NAME, op, funct3, funct7) \
- void NAME(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = relaxed) { \
- unsigned insn = 0; \
- uint32_t val = memory_order & 0x3; \
- patch((address)&insn, 6, 0, op); \
- patch((address)&insn, 14, 12, funct3); \
- patch_reg((address)&insn, 7, Rd); \
- patch_reg((address)&insn, 15, Rs2); \
- patch_reg((address)&insn, 20, Rs1); \
- patch((address)&insn, 31, 27, funct7); \
- patch((address)&insn, 26, 25, val); \
- emit(insn); \
+ enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11};
+
+ private:
+
+ enum AmoWidthFunct3 : uint8_t {
+ AMO_WIDTH_BYTE = 0b000, // Zabha extension
+ AMO_WIDTH_HALFWORD = 0b001, // Zabha extension
+ AMO_WIDTH_WORD = 0b010,
+ AMO_WIDTH_DOUBLEWORD = 0b011,
+ AMO_WIDTH_QUADWORD = 0b100,
+ // 0b101 to 0b111 are reserved
+ };
+
+ enum AmoOperationFunct5 : uint8_t {
+ AMO_ADD = 0b00000,
+ AMO_SWAP = 0b00001,
+ AMO_LR = 0b00010,
+ AMO_SC = 0b00011,
+ AMO_XOR = 0b00100,
+ AMO_OR = 0b01000,
+ AMO_AND = 0b01100,
+ AMO_MIN = 0b10000,
+ AMO_MAX = 0b10100,
+ AMO_MINU = 0b11000,
+ AMO_MAXU = 0b11100,
+ AMO_CAS = 0b00101 // Zacas
+ };
+
+ static constexpr uint32_t OP_AMO_MAJOR = 0b0101111;
+
+ template <AmoOperationFunct5 funct5, AmoWidthFunct3 width>
+ void amo_base(Register Rd, Register Rs1, uint8_t Rs2, Aqrl memory_order = aqrl) {
+ assert(width > AMO_WIDTH_HALFWORD || UseZabha, "Must be");
+ assert(funct5 != AMO_CAS || UseZacas, "Must be");
+ unsigned insn = 0;
+ patch((address)&insn, 6, 0, OP_AMO_MAJOR);
+ patch_reg((address)&insn, 7, Rd);
+ patch((address)&insn, 14, 12, width);
+ patch_reg((address)&insn, 15, Rs1);
+ patch((address)&insn, 24, 20, Rs2);
+ patch((address)&insn, 26, 25, memory_order);
+ patch((address)&insn, 31, 27, funct5);
+ emit(insn);
}
- INSN(sc_w, 0b0101111, 0b010, 0b00011);
- INSN(sc_d, 0b0101111, 0b011, 0b00011);
-#undef INSN
+ template <AmoOperationFunct5 funct5, AmoWidthFunct3 width>
+ void amo_base(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<funct5, width>(Rd, Rs1, Rs2->raw_encoding(), memory_order);
+ }
+
+ public:
+
+ void amoadd_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_ADD, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoadd_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_ADD, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoadd_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_ADD, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoadd_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_ADD, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoswap_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_SWAP, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoswap_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_SWAP, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoswap_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_SWAP, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoswap_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_SWAP, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoxor_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_XOR, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoxor_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_XOR, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoxor_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_XOR, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoxor_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_XOR, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoor_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_OR, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoor_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_OR, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoor_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_OR, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoor_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_OR, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoand_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_AND, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoand_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_AND, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoand_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_AND, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amoand_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_AND, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomin_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MIN, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomin_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MIN, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomin_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MIN, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomin_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MIN, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amominu_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MINU, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amominu_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MINU, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amominu_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MINU, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amominu_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MINU, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomax_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MAX, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomax_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MAX, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomax_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MAX, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomax_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MAX, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomaxu_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MAXU, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomaxu_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MAXU, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomaxu_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MAXU, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amomaxu_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_MAXU, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ protected:
+
+ void lr_w(Register Rd, Register Rs1, Aqrl memory_order = aqrl) {
+ amo_base<AMO_LR, AMO_WIDTH_WORD>(Rd, Rs1, 0, memory_order);
+ }
+
+ void lr_d(Register Rd, Register Rs1, Aqrl memory_order = aqrl) {
+ amo_base<AMO_LR, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, 0, memory_order);
+ }
+
+ void sc_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_SC, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void sc_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_SC, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amocas_b(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_CAS, AMO_WIDTH_BYTE>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amocas_h(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_CAS, AMO_WIDTH_HALFWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amocas_w(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_CAS, AMO_WIDTH_WORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ void amocas_d(Register Rd, Register Rs1, Register Rs2, Aqrl memory_order = aqrl) {
+ amo_base<AMO_CAS, AMO_WIDTH_DOUBLEWORD>(Rd, Rs1, Rs2, memory_order);
+ }
+
+ public:
+
+ enum operand_size { int8, int16, int32, uint32, int64 };
#define INSN(NAME, op, funct5, funct7) \
void NAME(FloatRegister Rd, FloatRegister Rs1, RoundingMode rm = rne) { \
@@ -286,7 +286,7 @@ void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm,
__ relocate(barrier_Relocation::spec(), [&] {
__ li16u(rtmp1, barrier_Relocation::unpatched);
}, ZBarrierRelocationFormatStoreGoodBits);
- __ cmpxchg_weak(rtmp2, zr, rtmp1,
+ __ weak_cmpxchg(rtmp2, zr, rtmp1,
Assembler::int64,
Assembler::relaxed /* acquire */, Assembler::relaxed /* release */,
rtmp3);
@@ -107,6 +107,7 @@ define_pd_global(intx, InlineSmallCode, 1000);
product(bool, UseZbb, false, DIAGNOSTIC, "Use Zbb instructions") \
product(bool, UseZbs, false, DIAGNOSTIC, "Use Zbs instructions") \
product(bool, UseZacas, false, EXPERIMENTAL, "Use Zacas instructions") \
+ product(bool, UseZabha, false, EXPERIMENTAL, "Use UseZabha instructions") \
product(bool, UseZfa, false, EXPERIMENTAL, "Use Zfa instructions") \
product(bool, UseZic64b, false, EXPERIMENTAL, "Use Zic64b instructions") \
product(bool, UseZicbom, false, EXPERIMENTAL, "Use Zicbom instructions") \
@@ -3284,7 +3284,7 @@ void MacroAssembler::cmpxchg_obj_header(Register oldv, Register newv, Register o
void MacroAssembler::load_reserved(Register dst,
Register addr,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire) {
switch (size) {
case int64:
@@ -3305,15 +3305,15 @@ void MacroAssembler::load_reserved(Register dst,
void MacroAssembler::store_conditional(Register dst,
Register new_val,
Register addr,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl release) {
switch (size) {
case int64:
- sc_d(dst, new_val, addr, release);
+ sc_d(dst, addr, new_val, release);
break;
case int32:
case uint32:
- sc_w(dst, new_val, addr, release);
+ sc_w(dst, addr, new_val, release);
break;
default:
ShouldNotReachHere();
@@ -3322,7 +3322,7 @@ void MacroAssembler::store_conditional(Register dst,
void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Register shift, Register mask, Register aligned_addr) {
assert(size == int8 || size == int16, "unsupported operand size");
@@ -3352,10 +3352,11 @@ void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expecte
// which are forced to work with 4-byte aligned address.
void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result, bool result_as_bool,
Register tmp1, Register tmp2, Register tmp3) {
+ assert(!(UseZacas && UseZabha), "Use amocas");
assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1);
Register scratch0 = t0, aligned_addr = t1;
@@ -3388,13 +3389,13 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
notr(scratch1, mask);
bind(retry);
- lr_w(result, aligned_addr, acquire);
+ load_reserved(result, aligned_addr, operand_size::int32, acquire);
andr(scratch0, result, mask);
bne(scratch0, expected, fail);
andr(scratch0, result, scratch1); // scratch1 is ~mask
orr(scratch0, scratch0, new_val);
- sc_w(scratch0, scratch0, aligned_addr, release);
+ store_conditional(scratch0, scratch0, aligned_addr, operand_size::int32, release);
bnez(scratch0, retry);
}
@@ -3426,10 +3427,11 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
// failed.
void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result,
Register tmp1, Register tmp2, Register tmp3) {
+ assert(!(UseZacas && UseZabha), "Use amocas");
assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1);
Register scratch0 = t0, aligned_addr = t1;
@@ -3460,13 +3462,13 @@ void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
} else {
notr(scratch1, mask);
- lr_w(result, aligned_addr, acquire);
+ load_reserved(result, aligned_addr, operand_size::int32, acquire);
andr(scratch0, result, mask);
bne(scratch0, expected, fail);
andr(scratch0, result, scratch1); // scratch1 is ~mask
orr(scratch0, scratch0, new_val);
- sc_w(scratch0, scratch0, aligned_addr, release);
+ store_conditional(scratch0, scratch0, aligned_addr, operand_size::int32, release);
bnez(scratch0, fail);
}
@@ -3483,10 +3485,10 @@ void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
void MacroAssembler::cmpxchg(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result, bool result_as_bool) {
- assert(size != int8 && size != int16, "unsupported operand size");
+ assert((UseZacas && UseZabha) || (size != int8 && size != int16), "unsupported operand size");
assert_different_registers(addr, t0);
assert_different_registers(expected, t0);
assert_different_registers(new_val, t0);
@@ -3542,12 +3544,12 @@ void MacroAssembler::cmpxchg(Register addr, Register expected,
bind(done);
}
-void MacroAssembler::cmpxchg_weak(Register addr, Register expected,
+void MacroAssembler::weak_cmpxchg(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result) {
-
+ assert((UseZacas && UseZabha) || (size != int8 && size != int16), "unsupported operand size");
assert_different_registers(addr, t0);
assert_different_registers(expected, t0);
assert_different_registers(new_val, t0);
@@ -3620,7 +3622,7 @@ ATOMIC_XCHGU(xchgalwu, xchgalw)
#undef ATOMIC_XCHGU
void MacroAssembler::atomic_cas(Register prev, Register newv, Register addr,
- enum operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) {
+ Assembler::operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) {
switch (size) {
case int64:
amocas_d(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
@@ -3632,6 +3634,12 @@ void MacroAssembler::atomic_cas(Register prev, Register newv, Register addr,
amocas_w(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
zero_extend(prev, prev, 32);
break;
+ case int16:
+ amocas_h(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
+ break;
+ case int8:
+ amocas_b(prev, addr, newv, (Assembler::Aqrl)(acquire | release));
+ break;
default:
ShouldNotReachHere();
}
@@ -1017,26 +1017,26 @@ public:
void cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp, Label &succeed, Label *fail);
void cmpxchg(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result, bool result_as_bool = false);
- void cmpxchg_weak(Register addr, Register expected,
+ void weak_cmpxchg(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result);
void cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Register shift, Register mask, Register aligned_addr);
void cmpxchg_narrow_value(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result, bool result_as_bool,
Register tmp1, Register tmp2, Register tmp3);
void weak_cmpxchg_narrow_value(Register addr, Register expected,
Register new_val,
- enum operand_size size,
+ Assembler::operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result,
Register tmp1, Register tmp2, Register tmp3);
@@ -1053,7 +1053,7 @@ public:
void atomic_xchgwu(Register prev, Register newv, Register addr);
void atomic_xchgalwu(Register prev, Register newv, Register addr);
- void atomic_cas(Register prev, Register newv, Register addr, enum operand_size size,
+ void atomic_cas(Register prev, Register newv, Register addr, Assembler::operand_size size,
Assembler::Aqrl acquire = Assembler::relaxed, Assembler::Aqrl release = Assembler::relaxed);
static bool far_branches() {
@@ -1508,8 +1508,8 @@ private:
int bitset_to_regs(unsigned int bitset, unsigned char* regs);
Address add_memory_helper(const Address dst, Register tmp);
- void load_reserved(Register dst, Register addr, enum operand_size size, Assembler::Aqrl acquire);
- void store_conditional(Register dst, Register new_val, Register addr, enum operand_size size, Assembler::Aqrl release);
+ void load_reserved(Register dst, Register addr, Assembler::operand_size size, Assembler::Aqrl acquire);
+ void store_conditional(Register dst, Register new_val, Register addr, Assembler::operand_size size, Assembler::Aqrl release);
public:
void lightweight_lock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow);
@@ -2256,48 +2256,6 @@ encode %{
}
%}
- enc_class riscv_enc_cmpxchgw(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{
- C2_MacroAssembler _masm(&cbuf);
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
- /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
- enc_class riscv_enc_cmpxchgn(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{
- C2_MacroAssembler _masm(&cbuf);
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
- /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
- enc_class riscv_enc_cmpxchg(iRegINoSp res, memory mem, iRegL oldval, iRegL newval) %{
- C2_MacroAssembler _masm(&cbuf);
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
- /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
- enc_class riscv_enc_cmpxchgw_acq(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{
- C2_MacroAssembler _masm(&cbuf);
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
- /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
- enc_class riscv_enc_cmpxchgn_acq(iRegINoSp res, memory mem, iRegI oldval, iRegI newval) %{
- C2_MacroAssembler _masm(&cbuf);
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
- /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
- enc_class riscv_enc_cmpxchg_acq(iRegINoSp res, memory mem, iRegL oldval, iRegL newval) %{
- C2_MacroAssembler _masm(&cbuf);
- __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
- /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
- /*result as bool*/ true);
- %}
-
// compare and branch instruction encodings
enc_class riscv_enc_j(label lbl) %{
@@ -5221,18 +5179,20 @@ instruct prefetchalloc( memory mem ) %{
// standard CompareAndSwapX when we are using barriers
// these have higher priority than the rules selected by a predicate
-instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndSwapB_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (CompareAndSwapB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 10 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
format %{
"cmpxchg $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t"
- "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapB"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapB_narrow"
%}
ins_encode %{
@@ -5244,18 +5204,42 @@ instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R1
ins_pipe(pipe_slow);
%}
-instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (CompareAndSwapB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapB"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
+ true /* result as bool */);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapS_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (CompareAndSwapS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 11 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
format %{
"cmpxchg $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t"
- "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapS"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapS_narrow"
%}
ins_encode %{
@@ -5267,18 +5251,44 @@ instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R1
ins_pipe(pipe_slow);
%}
+instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (CompareAndSwapS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapS"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
+ true /* result as bool */);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
match(Set res (CompareAndSwapI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapI"
%}
- ins_encode(riscv_enc_cmpxchgw(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5287,14 +5297,18 @@ instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval
%{
match(Set res (CompareAndSwapL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapL"
%}
- ins_encode(riscv_enc_cmpxchg(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5305,14 +5319,18 @@ instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapP"
%}
- ins_encode(riscv_enc_cmpxchg(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5321,33 +5339,37 @@ instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval
%{
match(Set res (CompareAndSwapN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 8 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapN"
%}
- ins_encode(riscv_enc_cmpxchgn(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
// alternative CompareAndSwapX when we are eliding barriers
-instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndSwapBAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (CompareAndSwapB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 10 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t"
- "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapBAcq"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapBAcq_narrow"
%}
ins_encode %{
@@ -5359,20 +5381,42 @@ instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI
ins_pipe(pipe_slow);
%}
-instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (CompareAndSwapB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $mem, $oldval, $newval\t# (byte) if $mem == $oldval then $mem <-- $newval\n\t"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapBAcq"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
+ true /* result as bool */);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapSAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+%{
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (CompareAndSwapS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 11 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t"
- "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapSAcq"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapSAcq_narrow"
%}
ins_encode %{
@@ -5384,20 +5428,46 @@ instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI
ins_pipe(pipe_slow);
%}
+instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (CompareAndSwapS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $mem, $oldval, $newval\t# (short) if $mem == $oldval then $mem <-- $newval\n\t"
+ "mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapSAcq"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
+ true /* result as bool */);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
predicate(needs_acquiring_load_reserved(n));
match(Set res (CompareAndSwapI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapIAcq"
%}
- ins_encode(riscv_enc_cmpxchgw_acq(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5408,14 +5478,18 @@ instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL new
match(Set res (CompareAndSwapL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapLAcq"
%}
- ins_encode(riscv_enc_cmpxchg_acq(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5426,14 +5500,18 @@ instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP new
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 6 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapPAcq"
%}
- ins_encode(riscv_enc_cmpxchg_acq(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5444,14 +5522,18 @@ instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN new
match(Set res (CompareAndSwapN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + ALU_COST * 8 + BRANCH_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval\n\t"
"mv $res, $res == $oldval\t# $res <-- ($res == $oldval ? 1 : 0), #@compareAndSwapNAcq"
%}
- ins_encode(riscv_enc_cmpxchgn_acq(res, mem, oldval, newval));
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
+ /*result as bool*/ true);
+ %}
ins_pipe(pipe_slow);
%}
@@ -5462,17 +5544,19 @@ instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN new
// no trailing StoreLoad barrier emitted by C2. Unfortunately we
// can't check the type of memory ordering here, so we always emit a
// sc_d(w) with rl bit set.
-instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndExchangeB_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 5);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeB"
+ "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeB_narrow"
%}
ins_encode %{
@@ -5484,17 +5568,39 @@ instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg
ins_pipe(pipe_slow);
%}
-instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeB"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeS_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 6);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeS"
+ "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeS_narrow"
%}
ins_encode %{
@@ -5506,13 +5612,31 @@ instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg
ins_pipe(pipe_slow);
%}
+instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeS"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeI"
@@ -5530,9 +5654,7 @@ instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL ne
%{
match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeL"
@@ -5550,9 +5672,7 @@ instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN ne
%{
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 3);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeN"
@@ -5569,11 +5689,10 @@ instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN ne
instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval)
%{
predicate(n->as_LoadStore()->barrier_data() == 0);
- match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
+ match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeP"
@@ -5587,19 +5706,19 @@ instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP ne
ins_pipe(pipe_slow);
%}
-instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndExchangeBAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 5);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeBAcq"
+ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeBAcq_narrow"
%}
ins_encode %{
@@ -5611,19 +5730,39 @@ instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i
ins_pipe(pipe_slow);
%}
-instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeBAcq"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeSAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+%{
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST * 6);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeSAcq"
+ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeSAcq_narrow"
%}
ins_encode %{
@@ -5635,15 +5774,33 @@ instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i
ins_pipe(pipe_slow);
%}
+instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "cmpxchg_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeSAcq"
+ %}
+
+ ins_encode %{
+ __ cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
predicate(needs_acquiring_load_reserved(n));
match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeIAcq"
@@ -5663,9 +5820,7 @@ instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL
match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeLAcq"
@@ -5685,9 +5840,7 @@ instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN
match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangeNAcq"
@@ -5707,9 +5860,7 @@ instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 3 + ALU_COST);
-
- effect(TEMP_DEF res);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
"cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval, #@compareAndExchangePAcq"
@@ -5723,18 +5874,20 @@ instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP
ins_pipe(pipe_slow);
%}
-instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct weakCompareAndSwapB_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 6);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
- "# $res == 1 when success, #@weakCompareAndSwapB"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapB_narrow"
%}
ins_encode %{
@@ -5746,18 +5899,41 @@ instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg
ins_pipe(pipe_slow);
%}
-instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "weak_cmpxchg $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapB"
+ %}
+
+ ins_encode %{
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapS_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
+ predicate(!UseZabha || !UseZacas);
+
match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 7);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
- "# $res == 1 when success, #@weakCompareAndSwapS"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapS_narrow"
%}
ins_encode %{
@@ -5769,19 +5945,40 @@ instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI_R12 oldval, iReg
ins_pipe(pipe_slow);
%}
+instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate(UseZabha && UseZacas);
+
+ match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "weak_cmpxchg $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapS"
+ %}
+
+ ins_encode %{
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapI"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
/*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5792,15 +5989,15 @@ instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL ne
%{
match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapL"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
/*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5811,15 +6008,15 @@ instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN ne
%{
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapN"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
/*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5829,37 +6026,38 @@ instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN ne
instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval)
%{
predicate(n->as_LoadStore()->barrier_data() == 0);
+
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
- "cmpxchg_weak $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapP"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
/*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
%}
ins_pipe(pipe_slow);
%}
-instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct weakCompareAndSwapBAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 6);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
- "# $res == 1 when success, #@weakCompareAndSwapBAcq"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapBAcq_narrow"
%}
ins_encode %{
@@ -5871,20 +6069,41 @@ instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i
ins_pipe(pipe_slow);
%}
-instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
- iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
- predicate(needs_acquiring_load_reserved(n));
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapBAcq"
+ %}
+
+ ins_encode %{
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int8,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapSAcq_narrow(iRegINoSp res, indirect mem, iRegI_R12 oldval, iRegI_R13 newval,
+ iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, rFlagsReg cr)
+%{
+ predicate((!UseZabha || !UseZacas) && needs_acquiring_load_reserved(n));
match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 7);
+ ins_cost(2 * VOLATILE_REF_COST);
effect(TEMP_DEF res, KILL cr, USE_KILL oldval, USE_KILL newval, TEMP tmp1, TEMP tmp2, TEMP tmp3);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
- "# $res == 1 when success, #@weakCompareAndSwapSAcq"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapSAcq_narrow"
%}
ins_encode %{
@@ -5896,21 +6115,42 @@ instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI_R12 oldval, i
ins_pipe(pipe_slow);
%}
+instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
+%{
+ predicate((UseZabha && UseZacas) && needs_acquiring_load_reserved(n));
+
+ match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
+
+ ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "# $res == 1 when success, #@weakCompareAndSwapSAcq"
+ %}
+
+ ins_encode %{
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int16,
+ /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval)
%{
predicate(needs_acquiring_load_reserved(n));
match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapIAcq"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int32,
/*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5923,15 +6163,15 @@ instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL
match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapLAcq"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
/*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5944,15 +6184,15 @@ instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN
match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 4);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"# $res == 1 when success, #@weakCompareAndSwapNAcq"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::uint32,
/*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
%}
@@ -5965,15 +6205,15 @@ instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
- ins_cost(LOAD_COST + STORE_COST + BRANCH_COST * 2 + ALU_COST * 2);
+ ins_cost(2 * VOLATILE_REF_COST);
format %{
- "cmpxchg_weak_acq $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t"
+ "weak_cmpxchg_acq $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval\n\t"
"\t# $res == 1 when success, #@weakCompareAndSwapPAcq"
%}
ins_encode %{
- __ cmpxchg_weak(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
+ __ weak_cmpxchg(as_Register($mem$$base), $oldval$$Register, $newval$$Register, Assembler::int64,
/*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
%}
