From 998f19d24fa496d638be1497e9e5a0c315d65dab Mon Sep 17 00:00:00 2001
Date: Thu, 27 Nov 2025 20:15:30 +0800
Subject: [PATCH 1/6] Add Compact Object Headers feature for Aarch64
make/Images.gmk | 20 +-
make/autoconf/configure.ac | 1 +
make/autoconf/jdk-options.m4 | 26 ++
make/autoconf/spec.gmk.in | 1 +
src/hotspot/cpu/aarch64/aarch64.ad | 53 ++-
.../cpu/aarch64/c1_CodeStubs_aarch64.cpp | 8 +
.../cpu/aarch64/c1_LIRAssembler_aarch64.cpp | 55 +--
.../cpu/aarch64/c1_MacroAssembler_aarch64.cpp | 55 ++-
.../cpu/aarch64/c1_MacroAssembler_aarch64.hpp | 2 +-
.../cpu/aarch64/c2_CodeStubs_aarch64.cpp | 11 +
.../cpu/aarch64/c2_MacroAssembler_aarch64.cpp | 328 ++++++++++++--
.../cpu/aarch64/c2_MacroAssembler_aarch64.hpp | 8 +-
src/hotspot/cpu/aarch64/globals_aarch64.hpp | 2 +
.../cpu/aarch64/interp_masm_aarch64.cpp | 19 +-
.../cpu/aarch64/macroAssembler_aarch64.cpp | 211 ++++++---
.../cpu/aarch64/macroAssembler_aarch64.hpp | 8 +-
.../cpu/aarch64/sharedRuntime_aarch64.cpp | 5 +-
.../cpu/aarch64/templateTable_aarch64.cpp | 28 +-
.../cpu/aarch64/vm_version_aarch64.hpp | 3 +-
src/hotspot/cpu/arm/c1_CodeStubs_arm.cpp | 4 +
src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp | 2 +-
src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp | 4 +
src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp | 2 +-
src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp | 18 +-
src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.hpp | 2 +-
src/hotspot/cpu/riscv/c1_CodeStubs_riscv.cpp | 5 +
.../cpu/riscv/c1_LIRAssembler_riscv.cpp | 2 +-
.../cpu/riscv/c1_MacroAssembler_riscv.cpp | 13 +-
.../cpu/riscv/c1_MacroAssembler_riscv.hpp | 2 +-
src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp | 5 +
src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp | 2 +-
.../cpu/s390/c1_MacroAssembler_s390.cpp | 10 +-
.../cpu/s390/c1_MacroAssembler_s390.hpp | 2 +-
src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp | 2 +-
src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp | 17 +-
src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp | 2 +-
src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp | 12 +
src/hotspot/share/c1/c1_CodeStubs.hpp | 18 +
src/hotspot/share/c1/c1_LIR.cpp | 7 +
src/hotspot/share/c1/c1_LIR.hpp | 7 +-
src/hotspot/share/c1/c1_LIRGenerator.cpp | 3 +-
src/hotspot/share/cds/archiveBuilder.cpp | 7 +
src/hotspot/share/cds/archiveHeapWriter.cpp | 40 +-
src/hotspot/share/cds/filemap.cpp | 10 +
src/hotspot/share/cds/filemap.hpp | 2 +
src/hotspot/share/ci/ciKlass.cpp | 22 +
src/hotspot/share/ci/ciKlass.hpp | 5 +
src/hotspot/share/gc/g1/g1CollectedHeap.cpp | 3 +
src/hotspot/share/gc/g1/g1FullCollector.cpp | 31 +-
src/hotspot/share/gc/g1/g1FullCollector.hpp | 4 +
.../share/gc/g1/g1FullGCAdjustTask.cpp | 42 +-
.../share/gc/g1/g1FullGCAdjustTask.hpp | 3 +-
.../share/gc/g1/g1FullGCCompactTask.cpp | 48 +-
.../share/gc/g1/g1FullGCCompactTask.hpp | 6 +
.../share/gc/g1/g1FullGCCompactionPoint.cpp | 19 +-
.../share/gc/g1/g1FullGCCompactionPoint.hpp | 2 +
.../share/gc/g1/g1FullGCOopClosures.hpp | 1 +
.../gc/g1/g1FullGCOopClosures.inline.hpp | 16 +-
.../share/gc/g1/g1FullGCPrepareTask.cpp | 17 +-
.../share/gc/g1/g1FullGCPrepareTask.hpp | 2 +
.../gc/g1/g1FullGCPrepareTask.inline.hpp | 10 +-
.../share/gc/g1/g1OopClosures.inline.hpp | 2 +-
.../share/gc/g1/g1ParScanThreadState.cpp | 28 +-
.../share/gc/g1/g1ParScanThreadState.hpp | 4 +-
.../share/gc/parallel/mutableSpace.cpp | 15 +-
src/hotspot/share/gc/parallel/psOldGen.cpp | 4 +-
.../share/gc/parallel/psPromotionManager.cpp | 4 +-
.../share/gc/parallel/psPromotionManager.hpp | 2 +-
.../gc/parallel/psPromotionManager.inline.hpp | 44 +-
.../share/gc/serial/defNewGeneration.cpp | 5 +-
src/hotspot/share/gc/serial/genMarkSweep.cpp | 35 +-
src/hotspot/share/gc/serial/markSweep.cpp | 29 +-
src/hotspot/share/gc/serial/markSweep.hpp | 13 +-
.../share/gc/serial/markSweep.inline.hpp | 24 +-
src/hotspot/share/gc/shared/collectedHeap.cpp | 15 +-
src/hotspot/share/gc/shared/collectedHeap.hpp | 2 +-
src/hotspot/share/gc/shared/gc_globals.hpp | 8 +-
.../share/gc/shared/genCollectedHeap.cpp | 3 +
src/hotspot/share/gc/shared/memAllocator.cpp | 16 +-
.../share/gc/shared/preservedMarks.cpp | 16 +-
.../share/gc/shared/preservedMarks.hpp | 3 +
.../share/gc/shared/preservedMarks.inline.hpp | 1 +
.../share/gc/shared/slidingForwarding.cpp | 123 +++++
.../share/gc/shared/slidingForwarding.hpp | 181 ++++++++
.../gc/shared/slidingForwarding.inline.hpp | 171 +++++++
src/hotspot/share/gc/shared/space.cpp | 50 +-
src/hotspot/share/gc/shared/space.hpp | 16 +-
.../share/gc/shenandoah/shenandoahAsserts.cpp | 4 +-
.../share/gc/shenandoah/shenandoahFullGC.cpp | 106 ++++-
.../share/gc/shenandoah/shenandoahFullGC.hpp | 5 +
.../share/gc/shenandoah/shenandoahHeap.cpp | 3 +
.../gc/shenandoah/shenandoahHeap.inline.hpp | 27 +-
.../gc/shenandoah/shenandoahVerifier.cpp | 17 +-
src/hotspot/share/gc/x/c2/xBarrierSetC2.cpp | 2 +-
src/hotspot/share/gc/x/xObjArrayAllocator.cpp | 23 +-
src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp | 2 +-
src/hotspot/share/gc/z/zObjArrayAllocator.cpp | 32 +-
src/hotspot/share/gc/z/zRelocate.cpp | 3 +-
.../interpreter/zero/bytecodeInterpreter.cpp | 11 +-
.../chains/objectSampleMarker.hpp | 2 +-
src/hotspot/share/jvmci/jvmciCompilerToVM.cpp | 2 +-
.../share/jvmci/jvmciCompilerToVMInit.cpp | 1 -
src/hotspot/share/jvmci/jvmci_globals.cpp | 1 -
src/hotspot/share/jvmci/jvmci_globals.hpp | 3 -
src/hotspot/share/jvmci/vmStructs_jvmci.cpp | 10 +
src/hotspot/share/memory/universe.cpp | 12 +-
src/hotspot/share/oops/arrayOop.hpp | 77 ++--
src/hotspot/share/oops/instanceOop.hpp | 17 +-
src/hotspot/share/oops/klass.cpp | 17 +
src/hotspot/share/oops/klass.hpp | 11 +
src/hotspot/share/oops/klass.inline.hpp | 7 +
src/hotspot/share/oops/markWord.hpp | 85 +++-
src/hotspot/share/oops/markWord.inline.hpp | 70 +++
src/hotspot/share/oops/objArrayKlass.cpp | 3 +-
.../share/oops/objArrayKlass.inline.hpp | 3 +-
src/hotspot/share/oops/objArrayOop.hpp | 37 +-
src/hotspot/share/oops/oop.cpp | 8 +-
src/hotspot/share/oops/oop.hpp | 68 ++-
src/hotspot/share/oops/oop.inline.hpp | 215 ++++++++-
src/hotspot/share/oops/typeArrayKlass.cpp | 3 +-
.../share/oops/typeArrayKlass.inline.hpp | 3 +-
src/hotspot/share/opto/c2_CodeStubs.hpp | 12 +-
src/hotspot/share/opto/callnode.cpp | 13 +-
src/hotspot/share/opto/compile.cpp | 6 +
src/hotspot/share/opto/library_call.cpp | 4 +-
src/hotspot/share/opto/macro.cpp | 4 +-
src/hotspot/share/opto/memnode.cpp | 20 +-
src/hotspot/share/opto/runtime.cpp | 13 +-
src/hotspot/share/opto/type.cpp | 4 +-
src/hotspot/share/prims/unsafe.cpp | 2 +-
src/hotspot/share/prims/whitebox.cpp | 11 +
.../share/runtime/abstract_vm_version.hpp | 5 +-
src/hotspot/share/runtime/arguments.cpp | 45 +-
src/hotspot/share/runtime/basicLock.cpp | 35 +-
src/hotspot/share/runtime/deoptimization.cpp | 6 +-
src/hotspot/share/runtime/globals.hpp | 1 +
src/hotspot/share/runtime/lockStack.cpp | 23 +-
src/hotspot/share/runtime/lockStack.hpp | 37 +-
.../share/runtime/lockStack.inline.hpp | 144 +++++-
src/hotspot/share/runtime/objectMonitor.cpp | 67 ++-
src/hotspot/share/runtime/objectMonitor.hpp | 5 +-
.../share/runtime/objectMonitor.inline.hpp | 8 +-
src/hotspot/share/runtime/sharedRuntime.cpp | 26 +-
src/hotspot/share/runtime/synchronizer.cpp | 264 +++++++----
src/hotspot/share/runtime/synchronizer.hpp | 23 +-
src/hotspot/share/runtime/vmStructs.cpp | 5 +
src/hotspot/share/utilities/fastHash.hpp | 97 ++++
.../share/utilities/globalDefinitions.hpp | 3 +-
.../jvm/hotspot/debugger/DebuggerBase.java | 13 +-
.../hotspot/debugger/MachineDescription.java | 5 +
.../debugger/MachineDescriptionAArch64.java | 4 +
.../sun/jvm/hotspot/memory/Universe.java | 7 -
.../classes/sun/jvm/hotspot/oops/Array.java | 41 +-
.../sun/jvm/hotspot/oops/Instance.java | 4 +-
.../classes/sun/jvm/hotspot/oops/Mark.java | 28 +-
.../classes/sun/jvm/hotspot/oops/Oop.java | 41 +-
.../classes/sun/jvm/hotspot/runtime/VM.java | 21 +
.../utilities/RobustOopDeterminator.java | 27 +-
.../gtest/gc/shared/test_preservedMarks.cpp | 3 +
.../gc/shared/test_slidingForwarding.cpp | 124 +++++
test/hotspot/gtest/oops/test_arrayOop.cpp | 67 ++-
test/hotspot/gtest/oops/test_objArrayOop.cpp | 69 +++
test/hotspot/gtest/oops/test_typeArrayOop.cpp | 6 +-
test/hotspot/gtest/runtime/test_lockStack.cpp | 427 ++++++++++++++++++
test/hotspot/jtreg/TEST.groups | 1 +
.../c2/irTests/TestVectorizationNotRun.java | 3 +-
.../lib/ir_framework/TestFramework.java | 3 +-
...tIndependentPacksWithCyclicDependency.java | 3 +-
.../jtreg/gc/TestAllocHumongousFragment.java | 12 +
.../jtreg/gc/g1/plab/TestPLABPromotion.java | 8 +-
.../stress/systemgc/TestSystemGCWithG1.java | 13 +-
.../systemgc/TestSystemGCWithSerial.java | 33 +-
.../systemgc/TestSystemGCWithShenandoah.java | 16 +
test/hotspot/jtreg/gtest/ArrayTests.java | 56 +++
test/hotspot/jtreg/gtest/LockStackGtests.java | 32 ++
test/hotspot/jtreg/gtest/ObjArrayTests.java | 85 ++++
.../runtime/FieldLayout/BaseOffsets.java | 130 ++++++
.../runtime/FieldLayout/OldLayoutCheck.java | 32 +-
.../cds/CdsDifferentCompactObjectHeaders.java | 66 +++
.../runtime/cds/appcds/TestZGCWithCDS.java | 15 +
...toCreateSharedArchiveNoDefaultArchive.java | 1 +
.../lockStack/TestLockStackCapacity.java | 108 +++++
test/jdk/com/sun/jdi/EATests.java | 157 +++++++
.../GetObjectSizeIntrinsicsTest.java | 36 +-
.../tools/jlink/plugins/CDSPluginTest.java | 21 +-
test/lib/jdk/test/whitebox/WhiteBox.java | 6 +-
186 files changed, 4768 insertions(+), 818 deletions(-)
create mode 100644 src/hotspot/share/gc/shared/slidingForwarding.cpp
create mode 100644 src/hotspot/share/gc/shared/slidingForwarding.hpp
create mode 100644 src/hotspot/share/gc/shared/slidingForwarding.inline.hpp
create mode 100644 src/hotspot/share/oops/markWord.inline.hpp
create mode 100644 src/hotspot/share/utilities/fastHash.hpp
create mode 100644 test/hotspot/gtest/gc/shared/test_slidingForwarding.cpp
create mode 100644 test/hotspot/gtest/oops/test_objArrayOop.cpp
create mode 100644 test/hotspot/gtest/runtime/test_lockStack.cpp
create mode 100644 test/hotspot/jtreg/gtest/ArrayTests.java
create mode 100644 test/hotspot/jtreg/gtest/LockStackGtests.java
create mode 100644 test/hotspot/jtreg/gtest/ObjArrayTests.java
create mode 100644 test/hotspot/jtreg/runtime/FieldLayout/BaseOffsets.java
create mode 100644 test/hotspot/jtreg/runtime/cds/CdsDifferentCompactObjectHeaders.java
create mode 100644 test/hotspot/jtreg/runtime/lockStack/TestLockStackCapacity.java
@@ -123,10 +123,16 @@ CDS_DUMP_FLAGS = -Xmx128M -Xms128M
# Helper function for creating the CDS archives for the JDK and JRE
#
# Param1 - VM variant (e.g., server, client, zero, ...)
-# Param2 - _nocoops, or empty
+# Param2 - _nocoops, _coh, _nocoops_coh, or empty
define CreateCDSArchive
- $1_$2_DUMP_EXTRA_ARG := $(if $(filter _nocoops, $2),-XX:-UseCompressedOops,)
- $1_$2_DUMP_TYPE := $(if $(filter _nocoops, $2),-NOCOOPS,)
+ $1_$2_COOPS_OPTION := $(if $(findstring _nocoops, $2),-XX:-UseCompressedOops)
+ # enable and also explicitly disable coh as needed.
+ ifeq ($(call isTargetCpuArch, aarch64), true)
+ $1_$2_COH_OPTION := -XX:+UnlockExperimentalVMOptions \
+ $(if $(findstring _coh, $2),-XX:+UseCompactObjectHeaders,-XX:-UseCompactObjectHeaders)
+ endif
+ $1_$2_DUMP_EXTRA_ARG := $$($1_$2_COOPS_OPTION) $$($1_$2_COH_OPTION)
+ $1_$2_DUMP_TYPE := $(if $(findstring _nocoops, $2),-NOCOOPS,)$(if $(findstring _coh, $2),-COH,)
# Only G1 supports dumping the shared heap, so explicitly use G1 if the JVM supports it.
$1_$2_CDS_DUMP_FLAGS := $(CDS_DUMP_FLAGS) $(if $(filter g1gc, $(JVM_FEATURES_$1)),-XX:+UseG1GC)
@@ -173,6 +179,14 @@ ifeq ($(BUILD_CDS_ARCHIVE), true)
$(foreach v, $(JVM_VARIANTS), \
$(eval $(call CreateCDSArchive,$v,_nocoops)) \
)
+ ifeq ($(BUILD_CDS_ARCHIVE_COH), true)
+ $(foreach v, $(JVM_VARIANTS), \
+ $(eval $(call CreateCDSArchive,$v,_coh)) \
+ )
+ $(foreach v, $(JVM_VARIANTS), \
+ $(eval $(call CreateCDSArchive,$v,_nocoops_coh)) \
+ )
+ endif
endif
endif
@@ -261,6 +261,7 @@ JDKOPT_ENABLE_DISABLE_GENERATE_CLASSLIST
JDKOPT_EXCLUDE_TRANSLATIONS
JDKOPT_ENABLE_DISABLE_MANPAGES
JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE
+JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE_COH
JDKOPT_ENABLE_DISABLE_COMPATIBLE_CDS_ALIGNMENT
JDKOPT_SETUP_MACOSX_SIGNING
@@ -704,6 +704,32 @@ AC_DEFUN([JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE],
AC_SUBST(BUILD_CDS_ARCHIVE)
])
+################################################################################
+#
+# Enable or disable the default CDS archive generation for Compact Object Headers
+#
+AC_DEFUN([JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE_COH],
+[
+ UTIL_ARG_ENABLE(NAME: cds-archive-coh, DEFAULT: auto, RESULT: BUILD_CDS_ARCHIVE_COH,
+ DESC: [enable generation of default CDS archives for compact object headers (requires --enable-cds-archive)],
+ DEFAULT_DESC: [auto],
+ CHECKING_MSG: [if default CDS archives for compact object headers should be generated],
+ CHECK_AVAILABLE: [
+ AC_MSG_CHECKING([if CDS archive with compact object headers is available])
+ if test "x$BUILD_CDS_ARCHIVE" = "xfalse"; then
+ AC_MSG_RESULT([no (CDS default archive generation is disabled)])
+ AVAILABLE=false
+ elif test "x$OPENJDK_TARGET_CPU" != "xaarch64"; then
+ AC_MSG_RESULT([no (compact object headers not supported for this platform)])
+ AVAILABLE=false
+ else
+ AC_MSG_RESULT([yes])
+ AVAILABLE=true
+ fi
+ ])
+ AC_SUBST(BUILD_CDS_ARCHIVE_COH)
+])
+
################################################################################
#
# Enable the alternative CDS core region alignment
@@ -368,6 +368,7 @@ EXCLUDE_TRANSLATIONS := @EXCLUDE_TRANSLATIONS@
BUILD_MANPAGES := @BUILD_MANPAGES@
BUILD_CDS_ARCHIVE := @BUILD_CDS_ARCHIVE@
+BUILD_CDS_ARCHIVE_COH := @BUILD_CDS_ARCHIVE_COH@
ENABLE_COMPATIBLE_CDS_ALIGNMENT := @ENABLE_COMPATIBLE_CDS_ALIGNMENT@
@@ -7142,7 +7142,7 @@ instruct loadKlass(iRegPNoSp dst, memory8 mem)
instruct loadNKlass(iRegNNoSp dst, memory4 mem)
%{
match(Set dst (LoadNKlass mem));
- predicate(!needs_acquiring_load(n));
+ predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
ins_cost(4 * INSN_COST);
format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
@@ -7152,6 +7152,20 @@ instruct loadNKlass(iRegNNoSp dst, memory4 mem)
ins_pipe(iload_reg_mem);
%}
+instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
+%{
+ match(Set dst (LoadNKlass mem));
+ effect(KILL cr);
+ predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
+
+ ins_cost(4 * INSN_COST);
+ format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
+ ins_encode %{
+ __ load_nklass_compact($dst$$Register, $mem$$base$$Register, $mem$$index$$Register, $mem$$scale, $mem$$disp);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
// Load Float
instruct loadF(vRegF dst, memory4 mem)
%{
@@ -16433,13 +16447,12 @@ instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
%{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastLock object box));
effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
- // TODO
- // identify correct cost
ins_cost(5 * INSN_COST);
- format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
+ format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
ins_encode %{
__ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
@@ -16450,6 +16463,7 @@ instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegP
instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
%{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastUnlock object box));
effect(TEMP tmp, TEMP tmp2);
@@ -16463,6 +16477,37 @@ instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRe
ins_pipe(pipe_serial);
%}
+instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
+%{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastLock object box));
+ effect(TEMP tmp, TEMP tmp2);
+
+ ins_cost(5 * INSN_COST);
+ format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
+
+ ins_encode %{
+ __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
+instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
+%{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastUnlock object box));
+ effect(TEMP tmp, TEMP tmp2);
+
+ ins_cost(5 * INSN_COST);
+ format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
+
+ ins_encode %{
+ __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
//
// Safepoint Instructions
@@ -32,6 +32,7 @@
#include "c1/c1_Runtime1.hpp"
#include "classfile/javaClasses.hpp"
#include "nativeInst_aarch64.hpp"
+#include "runtime/objectMonitor.hpp"
#include "runtime/sharedRuntime.hpp"
#include "vmreg_aarch64.inline.hpp"
@@ -233,6 +234,13 @@ void MonitorExitStub::emit_code(LIR_Assembler* ce) {
__ far_jump(RuntimeAddress(Runtime1::entry_for(exit_id)));
}
+void LoadKlassStub::emit_code(LIR_Assembler* ce) {
+ assert(UseCompactObjectHeaders, "Only use with compact object headers");
+ __ bind(_entry);
+ Register d = _result->as_register();
+ __ ldr(d, Address(d, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
+ __ b(_continuation);
+}
// Implementation of patching:
// - Copy the code at given offset to an inlined buffer (first the bytes, then the number of bytes)
@@ -1230,7 +1230,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
len,
tmp1,
tmp2,
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
array_element_size(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -2290,8 +2290,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
- Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
- Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
// test for null
if (flags & LIR_OpArrayCopy::src_null_check) {
@@ -2352,15 +2350,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// We don't know the array types are compatible
if (basic_type != T_OBJECT) {
// Simple test for basic type arrays
- if (UseCompressedClassPointers) {
- __ ldrw(tmp, src_klass_addr);
- __ ldrw(rscratch1, dst_klass_addr);
- __ cmpw(tmp, rscratch1);
- } else {
- __ ldr(tmp, src_klass_addr);
- __ ldr(rscratch1, dst_klass_addr);
- __ cmp(tmp, rscratch1);
- }
+ __ cmp_klass(src, dst, tmp, rscratch1);
__ br(Assembler::NE, *stub->entry());
} else {
// For object arrays, if src is a sub class of dst then we can
@@ -2482,36 +2472,14 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// but not necessarily exactly of type default_type.
Label known_ok, halt;
__ mov_metadata(tmp, default_type->constant_encoding());
- if (UseCompressedClassPointers) {
- __ encode_klass_not_null(tmp);
- }
if (basic_type != T_OBJECT) {
-
- if (UseCompressedClassPointers) {
- __ ldrw(rscratch1, dst_klass_addr);
- __ cmpw(tmp, rscratch1);
- } else {
- __ ldr(rscratch1, dst_klass_addr);
- __ cmp(tmp, rscratch1);
- }
+ __ cmp_klass(dst, tmp, rscratch1);
__ br(Assembler::NE, halt);
- if (UseCompressedClassPointers) {
- __ ldrw(rscratch1, src_klass_addr);
- __ cmpw(tmp, rscratch1);
- } else {
- __ ldr(rscratch1, src_klass_addr);
- __ cmp(tmp, rscratch1);
- }
+ __ cmp_klass(src, tmp, rscratch1);
__ br(Assembler::EQ, known_ok);
} else {
- if (UseCompressedClassPointers) {
- __ ldrw(rscratch1, dst_klass_addr);
- __ cmpw(tmp, rscratch1);
- } else {
- __ ldr(rscratch1, dst_klass_addr);
- __ cmp(tmp, rscratch1);
- }
+ __ cmp_klass(dst, tmp, rscratch1);
__ br(Assembler::EQ, known_ok);
__ cmp(src, dst);
__ br(Assembler::EQ, known_ok);
@@ -2593,7 +2561,18 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
}
if (UseCompressedClassPointers) {
- __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
+ if (UseCompactObjectHeaders) {
+ // Check if we can take the (common) fast path, if obj is unlocked.
+ __ ldr(result, Address(obj, oopDesc::mark_offset_in_bytes()));
+ __ tst(result, markWord::monitor_value);
+ __ br(Assembler::NE, *op->stub()->entry());
+ __ bind(*op->stub()->continuation());
+
+ // Shift to get proper narrow Klass*.
+ __ lsr(result, result, markWord::klass_shift);
+ } else {
+ __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
+ }
__ decode_klass_not_null(result);
} else {
__ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -80,12 +80,12 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
br(Assembler::NE, slow_case);
}
- // Load object header
- ldr(hdr, Address(obj, hdr_offset));
if (LockingMode == LM_LIGHTWEIGHT) {
lightweight_lock(obj, hdr, temp, rscratch2, slow_case);
} else if (LockingMode == LM_LEGACY) {
Label done;
+ // Load object header
+ ldr(hdr, Address(obj, hdr_offset));
// and mark it as unlocked
orr(hdr, hdr, markWord::unlocked_value);
// save unlocked object header into the displaced header location on the stack
@@ -144,11 +144,6 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
verify_oop(obj);
if (LockingMode == LM_LIGHTWEIGHT) {
- ldr(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
- // We cannot use tbnz here, the target might be too far away and cannot
- // be encoded.
- tst(hdr, markWord::monitor_value);
- br(Assembler::NE, slow_case);
lightweight_unlock(obj, hdr, temp, rscratch2, slow_case);
} else if (LockingMode == LM_LEGACY) {
// test if object header is pointing to the displaced header, and if so, restore
@@ -180,20 +175,25 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
assert_different_registers(obj, klass, len);
- // This assumes that all prototype bits fit in an int32_t
- mov(t1, (int32_t)(intptr_t)markWord::prototype().value());
- str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
-
- if (UseCompressedClassPointers) { // Take care not to kill klass
- encode_klass_not_null(t1, klass);
- strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
+ if (UseCompactObjectHeaders) {
+ ldr(t1, Address(klass, Klass::prototype_header_offset()));
+ str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
} else {
- str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ // This assumes that all prototype bits fit in an int32_t
+ mov(t1, (int32_t)(intptr_t)markWord::prototype().value());
+ str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ if (UseCompressedClassPointers) { // Take care not to kill klass
+ encode_klass_not_null(t1, klass);
+ strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
+ } else {
+ str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
+ }
}
if (len->is_valid()) {
strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
- } else if (UseCompressedClassPointers) {
+ } else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
store_klass_gap(obj, zr);
}
}
@@ -271,7 +271,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
verify_oop(obj);
}
-void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) {
+void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
assert_different_registers(obj, len, t1, t2, klass);
// determine alignment mask
@@ -284,7 +284,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
const Register arr_size = t2; // okay to be the same
// align object end
- mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+ mov(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
add(arr_size, arr_size, len, ext::uxtw, f);
andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
@@ -292,8 +292,19 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
initialize_header(obj, klass, len, t1, t2);
+ // Clear leading 4 bytes, if necessary.
+ // TODO: This could perhaps go into initialize_body() and also clear the leading 4 bytes
+ // for non-array objects, thereby replacing the klass-gap clearing code in initialize_header().
+ int base_offset = base_offset_in_bytes;
+ if (!is_aligned(base_offset, BytesPerWord)) {
+ assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
+ strw(zr, Address(obj, base_offset));
+ base_offset += BytesPerInt;
+ }
+ assert(is_aligned(base_offset, BytesPerWord), "must be word-aligned");
+
// clear rest of allocated space
- initialize_body(obj, arr_size, header_size * BytesPerWord, t1, t2);
+ initialize_body(obj, arr_size, base_offset, t1, t2);
if (Compilation::current()->bailed_out()) {
return;
}
@@ -312,9 +323,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
verify_oop(receiver);
// explicit null check not needed since load from [klass_offset] causes a trap
- // check against inline cache
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
-
+ // check against inline cache. This is checked in Universe::genesis()..
cmp_klass(receiver, iCache, rscratch1);
}
@@ -100,7 +100,7 @@ using MacroAssembler::null_check;
// header_size: size of object header in words
// f : element scale factor
// slow_case : exit to slow case implementation if fast allocation fails
- void allocate_array(Register obj, Register len, Register t, Register t2, int header_size, int f, Register klass, Label& slow_case);
+ void allocate_array(Register obj, Register len, Register t, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case);
int rsp_offset() const { return _rsp_offset; }
void set_rsp_offset(int n) { _rsp_offset = n; }
@@ -91,4 +91,15 @@ void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
__ b(continuation());
}
+int C2LoadNKlassStub::max_size() const {
+ return 8;
+}
+
+void C2LoadNKlassStub::emit(C2_MacroAssembler& masm) {
+ __ bind(entry());
+ Register d = dst();
+ __ ldr(d, Address(d, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
+ __ b(continuation());
+}
+
#undef __
@@ -33,6 +33,7 @@
#include "opto/subnode.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/powerOfTwo.hpp"
+#include "utilities/globalDefinitions.hpp"
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
@@ -157,6 +158,7 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
Label object_has_monitor;
Label count, no_count;
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
assert_different_registers(oop, box, tmp, disp_hdr);
// Load markWord from object into displaced_header.
@@ -175,7 +177,8 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
if (LockingMode == LM_MONITOR) {
tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
b(cont);
- } else if (LockingMode == LM_LEGACY) {
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Set tmp to be (markWord of object | UNLOCK_VALUE).
orr(tmp, disp_hdr, markWord::unlocked_value);
@@ -204,10 +207,6 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
b(cont);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- lightweight_lock(oop, disp_hdr, tmp, tmp3Reg, no_count);
- b(count);
}
// Handle existing monitor.
@@ -221,14 +220,13 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
/*release*/ true, /*weak*/ false, tmp3Reg); // Sets flags for result
- if (LockingMode != LM_LIGHTWEIGHT) {
- // Store a non-null value into the box to avoid looking like a re-entrant
- // lock. The fast-path monitor unlock code checks for
- // markWord::monitor_value so use markWord::unused_mark which has the
- // relevant bit set, and also matches ObjectSynchronizer::enter.
- mov(tmp, (address)markWord::unused_mark().value());
- str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
- }
+ // Store a non-null value into the box to avoid looking like a re-entrant
+ // lock. The fast-path monitor unlock code checks for
+ // markWord::monitor_value so use markWord::unused_mark which has the
+ // relevant bit set, and also matches ObjectSynchronizer::enter.
+ mov(tmp, (address)markWord::unused_mark().value());
+ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
+
br(Assembler::EQ, cont); // CAS success means locking succeeded
cmp(tmp3Reg, rthread);
@@ -259,6 +257,7 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
Label object_has_monitor;
Label count, no_count;
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
assert_different_registers(oop, box, tmp, disp_hdr);
if (LockingMode == LM_LEGACY) {
@@ -277,7 +276,8 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
if (LockingMode == LM_MONITOR) {
tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
b(cont);
- } else if (LockingMode == LM_LEGACY) {
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Check if it is still a light weight lock, this is is true if we
// see the stack address of the basicLock in the markWord of the
// object.
@@ -285,10 +285,6 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
/*release*/ true, /*weak*/ false, tmp);
b(cont);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- lightweight_unlock(oop, tmp, box, disp_hdr, no_count);
- b(count);
}
assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
@@ -298,19 +294,6 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
add(tmp, tmp, -(int)markWord::monitor_value); // monitor
- if (LockingMode == LM_LIGHTWEIGHT) {
- // If the owner is anonymous, we need to fix it -- in an outline stub.
- Register tmp2 = disp_hdr;
- ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
- // We cannot use tbnz here, the target might be too far away and cannot
- // be encoded.
- tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
- C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
- Compile::current()->output()->add_stub(stub);
- br(Assembler::NE, stub->entry());
- bind(stub->continuation());
- }
-
ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
Label notRecursive;
@@ -343,6 +326,262 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
bind(no_count);
}
+void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
+ Register t2, Register t3) {
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert_different_registers(obj, t1, t2, t3);
+
+ // Handle inflated monitor.
+ Label inflated;
+ // Finish fast lock successfully. MUST branch to with flag == EQ
+ Label locked;
+ // Finish fast lock unsuccessfully. MUST branch to with flag == NE
+ Label slow_path;
+
+ if (DiagnoseSyncOnValueBasedClasses != 0) {
+ load_klass(t1, obj);
+ ldrw(t1, Address(t1, Klass::access_flags_offset()));
+ tstw(t1, JVM_ACC_IS_VALUE_BASED_CLASS);
+ br(Assembler::NE, slow_path);
+ }
+
+ const Register t1_mark = t1;
+
+ { // Lightweight locking
+
+ // Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
+ Label push;
+
+ const Register t2_top = t2;
+ const Register t3_t = t3;
+
+ // Check if lock-stack is full.
+ ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ cmpw(t2_top, (unsigned)LockStack::end_offset() - 1);
+ br(Assembler::GT, slow_path);
+
+ // Check if recursive.
+ subw(t3_t, t2_top, oopSize);
+ ldr(t3_t, Address(rthread, t3_t));
+ cmp(obj, t3_t);
+ br(Assembler::EQ, push);
+
+ // Relaxed normal load to check for monitor. Optimization for monitor case.
+ ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
+
+ // Not inflated
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
+
+ // Try to lock. Transition lock-bits 0b01 => 0b00
+ orr(t1_mark, t1_mark, markWord::unlocked_value);
+ eor(t3_t, t1_mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
+ /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
+ br(Assembler::NE, slow_path);
+
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ str(obj, Address(rthread, t2_top));
+ addw(t2_top, t2_top, oopSize);
+ strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ b(locked);
+ }
+
+ { // Handle inflated monitor.
+ bind(inflated);
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register t1_tagged_monitor = t1_mark;
+ const uintptr_t monitor_tag = markWord::monitor_value;
+ const Register t2_owner_addr = t2;
+ const Register t3_owner = t3;
+
+ // Compute owner address.
+ lea(t2_owner_addr, Address(t1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
+
+ // CAS owner (null => current thread).
+ cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
+ /*release*/ false, /*weak*/ false, t3_owner);
+ br(Assembler::EQ, locked);
+
+ // Check if recursive.
+ cmp(t3_owner, rthread);
+ br(Assembler::NE, slow_path);
+
+ // Recursive.
+ increment(Address(t1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1);
+ }
+
+ bind(locked);
+ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
+
+#ifdef ASSERT
+ // Check that locked label is reached with Flags == EQ.
+ Label flag_correct;
+ br(Assembler::EQ, flag_correct);
+ stop("Fast Lock Flag != EQ");
+#endif
+
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with Flags == NE.
+ br(Assembler::NE, flag_correct);
+ stop("Fast Lock Flag != NE");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
+}
+
+void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Register t2,
+ Register t3) {
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert_different_registers(obj, t1, t2, t3);
+
+ // Handle inflated monitor.
+ Label inflated, inflated_load_monitor;
+ // Finish fast unlock successfully. MUST branch to with flag == EQ
+ Label unlocked;
+ // Finish fast unlock unsuccessfully. MUST branch to with flag == NE
+ Label slow_path;
+
+ const Register t1_mark = t1;
+ const Register t2_top = t2;
+ const Register t3_t = t3;
+
+ { // Lightweight unlock
+
+ // Check if obj is top of lock-stack.
+ ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ subw(t2_top, t2_top, oopSize);
+ ldr(t3_t, Address(rthread, t2_top));
+ cmp(obj, t3_t);
+ // Top of lock stack was not obj. Must be monitor.
+ br(Assembler::NE, inflated_load_monitor);
+
+ // Pop lock-stack.
+ DEBUG_ONLY(str(zr, Address(rthread, t2_top));)
+ strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+
+ // Check if recursive.
+ subw(t3_t, t2_top, oopSize);
+ ldr(t3_t, Address(rthread, t3_t));
+ cmp(obj, t3_t);
+ br(Assembler::EQ, unlocked);
+
+ // Not recursive.
+ // Load Mark.
+ ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Check header for monitor (0b10).
+ tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
+ orr(t3_t, t1_mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
+ /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
+ br(Assembler::EQ, unlocked);
+
+ // Compare and exchange failed.
+ // Restore lock-stack and handle the unlock in runtime.
+ DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
+ addw(t2_top, t2_top, oopSize);
+ str(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ b(slow_path);
+ }
+
+
+ { // Handle inflated monitor.
+ bind(inflated_load_monitor);
+ ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+#ifdef ASSERT
+ tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
+ stop("Fast Unlock not monitor");
+#endif
+
+ bind(inflated);
+
+#ifdef ASSERT
+ Label check_done;
+ subw(t2_top, t2_top, oopSize);
+ cmpw(t2_top, in_bytes(JavaThread::lock_stack_base_offset()));
+ br(Assembler::LT, check_done);
+ ldr(t3_t, Address(rthread, t2_top));
+ cmp(obj, t3_t);
+ br(Assembler::NE, inflated);
+ stop("Fast Unlock lock on stack");
+ bind(check_done);
+#endif
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register t1_monitor = t1_mark;
+ const uintptr_t monitor_tag = markWord::monitor_value;
+
+ // Untag the monitor.
+ sub(t1_monitor, t1_mark, monitor_tag);
+
+ const Register t2_recursions = t2;
+ Label not_recursive;
+
+ // Check if recursive.
+ ldr(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
+ cbz(t2_recursions, not_recursive);
+
+ // Recursive unlock.
+ sub(t2_recursions, t2_recursions, 1u);
+ str(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
+ // Set flag == EQ
+ cmp(t2_recursions, t2_recursions);
+ b(unlocked);
+
+ bind(not_recursive);
+
+ Label release;
+ const Register t2_owner_addr = t2;
+
+ // Compute owner address.
+ lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
+
+ // Check if the entry lists are empty.
+ ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
+ ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
+ orr(rscratch1, rscratch1, t3_t);
+ cmp(rscratch1, zr);
+ br(Assembler::EQ, release);
+
+ // The owner may be anonymous and we removed the last obj entry in
+ // the lock-stack. This loses the information about the owner.
+ // Write the thread to the owner field so the runtime knows the owner.
+ str(rthread, Address(t2_owner_addr));
+ b(slow_path);
+
+ bind(release);
+ // Set owner to null.
+ // Release to satisfy the JMM
+ stlr(zr, t2_owner_addr);
+ }
+
+ bind(unlocked);
+ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
+
+#ifdef ASSERT
+ // Check that unlocked label is reached with Flags == EQ.
+ Label flag_correct;
+ br(Assembler::EQ, flag_correct);
+ stop("Fast Unlock Flag != EQ");
+#endif
+
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with Flags == NE.
+ br(Assembler::NE, flag_correct);
+ stop("Fast Unlock Flag != NE");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
+}
+
// Search for str1 in str2 and return index or -1
// Clobbers: rscratch1, rscratch2, rflags. May also clobber v0-v1, when icnt1==-1.
void C2_MacroAssembler::string_indexof(Register str2, Register str1,
@@ -2356,3 +2595,30 @@ bool C2_MacroAssembler::in_scratch_emit_size() {
}
return MacroAssembler::in_scratch_emit_size();
}
+
+void C2_MacroAssembler::load_nklass_compact(Register dst, Register obj, Register index, int scale, int disp) {
+ C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
+ Compile::current()->output()->add_stub(stub);
+
+ // Note: Don't clobber obj anywhere in that method!
+
+ // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
+ // obj-start, so that we can load from the object's mark-word instead. Usually the address
+ // comes as obj-start in obj and klass_offset_in_bytes in disp. However, sometimes C2
+ // emits code that pre-computes obj-start + klass_offset_in_bytes into a register, and
+ // then passes that register as obj and 0 in disp. The following code extracts the base
+ // and offset to load the mark-word.
+ int offset = oopDesc::mark_offset_in_bytes() + disp - oopDesc::klass_offset_in_bytes();
+ if (index == noreg) {
+ ldr(dst, Address(obj, offset));
+ } else {
+ lea(dst, Address(obj, index, Address::lsl(scale)));
+ ldr(dst, Address(dst, offset));
+ }
+ // NOTE: We can't use tbnz here, because the target is sometimes too far away
+ // and cannot be encoded.
+ tst(dst, markWord::monitor_value);
+ br(Assembler::NE, stub->entry());
+ bind(stub->continuation());
+ lsr(dst, dst, markWord::klass_shift);
+}
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -43,9 +43,11 @@
BasicType eltype);
// Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
- // See full description in macroAssembler_aarch64.cpp.
void fast_lock(Register object, Register box, Register tmp, Register tmp2, Register tmp3);
void fast_unlock(Register object, Register box, Register tmp, Register tmp2);
+ // Code used by cmpFastLockLightweight and cmpFastUnlockLightweight mach instructions in .ad file.
+ void fast_lock_lightweight(Register object, Register t1, Register t2, Register t3);
+ void fast_unlock_lightweight(Register object, Register t1, Register t2, Register t3);
void string_compare(Register str1, Register str2,
Register cnt1, Register cnt2, Register result,
@@ -184,4 +186,6 @@
void vector_signum_sve(FloatRegister dst, FloatRegister src, FloatRegister zero,
FloatRegister one, FloatRegister vtmp, PRegister pgtmp, SIMD_RegVariant T);
+ void load_nklass_compact(Register dst, Register obj, Register index, int scale, int disp);
+
#endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP
@@ -107,6 +107,8 @@ define_pd_global(intx, InlineSmallCode, 1000);
product(uint, UseSVE, 0, \
"Highest supported SVE instruction set version") \
range(0, 2) \
+ product(bool, UseCompactObjectHeaders, false, EXPERIMENTAL, \
+ "Use compact 64-bit object headers in 64-bit VM") \
product(bool, UseBlockZeroing, true, \
"Use DC ZVA for block zeroing") \
product(intx, BlockZeroingLowLimit, 256, \
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -767,7 +767,6 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
}
if (LockingMode == LM_LIGHTWEIGHT) {
- ldr(tmp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
lightweight_lock(obj_reg, tmp, tmp2, tmp3, slow_case);
b(count);
} else if (LockingMode == LM_LEGACY) {
@@ -884,22 +883,6 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
if (LockingMode == LM_LIGHTWEIGHT) {
Label slow_case;
-
- // Check for non-symmetric locking. This is allowed by the spec and the interpreter
- // must handle it.
- Register tmp = rscratch1;
- // First check for lock-stack underflow.
- ldrw(tmp, Address(rthread, JavaThread::lock_stack_top_offset()));
- cmpw(tmp, (unsigned)LockStack::start_offset());
- br(Assembler::LE, slow_case);
- // Then check if the top of the lock-stack matches the unlocked object.
- subw(tmp, tmp, oopSize);
- ldr(tmp, Address(rthread, tmp));
- cmpoop(tmp, obj_reg);
- br(Assembler::NE, slow_case);
-
- ldr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- tbnz(header_reg, exact_log2(markWord::monitor_value), slow_case);
lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
b(count);
bind(slow_case);
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -23,8 +23,6 @@
*
*/
-#include <sys/types.h>
-
#include "precompiled.hpp"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
@@ -54,6 +52,7 @@
#include "runtime/jniHandles.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
+#include "utilities/globalDefinitions.hpp"
#include "utilities/powerOfTwo.hpp"
#ifdef COMPILER1
#include "c1/c1_LIRAssembler.hpp"
@@ -65,6 +64,8 @@
#include "opto/output.hpp"
#endif
+#include <sys/types.h>
+
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
#else
@@ -4664,8 +4665,30 @@ void MacroAssembler::load_method_holder(Register holder, Register method) {
ldr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass*
}
+// Loads the obj's Klass* into dst.
+// Preserves all registers (incl src, rscratch1 and rscratch2).
+void MacroAssembler::load_nklass_compact(Register dst, Register src) {
+ assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
+
+ Label fast;
+
+ // Check if we can take the (common) fast path, if obj is unlocked.
+ ldr(dst, Address(src, oopDesc::mark_offset_in_bytes()));
+ tbz(dst, exact_log2(markWord::monitor_value), fast);
+
+ // Fetch displaced header
+ ldr(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
+
+ // Fast-path: shift to get narrowKlass.
+ bind(fast);
+ lsr(dst, dst, markWord::klass_shift);
+}
+
void MacroAssembler::load_klass(Register dst, Register src) {
- if (UseCompressedClassPointers) {
+ if (UseCompactObjectHeaders) {
+ load_nklass_compact(dst, src);
+ decode_klass_not_null(dst);
+ } else if (UseCompressedClassPointers) {
ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
decode_klass_not_null(dst);
} else {
@@ -4721,8 +4744,13 @@ void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, R
}
void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
+ assert_different_registers(oop, trial_klass, tmp);
if (UseCompressedClassPointers) {
- ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
+ if (UseCompactObjectHeaders) {
+ load_nklass_compact(tmp, oop);
+ } else {
+ ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
+ }
if (CompressedKlassPointers::base() == nullptr) {
cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
return;
@@ -4739,9 +4767,26 @@ void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp)
cmp(trial_klass, tmp);
}
+void MacroAssembler::cmp_klass(Register src, Register dst, Register tmp1, Register tmp2) {
+ if (UseCompactObjectHeaders) {
+ load_nklass_compact(tmp1, src);
+ load_nklass_compact(tmp2, dst);
+ cmpw(tmp1, tmp2);
+ } else if (UseCompressedClassPointers) {
+ ldrw(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
+ ldrw(tmp2, Address(dst, oopDesc::klass_offset_in_bytes()));
+ cmpw(tmp1, tmp2);
+ } else {
+ ldr(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
+ ldr(tmp2, Address(dst, oopDesc::klass_offset_in_bytes()));
+ cmp(tmp1, tmp2);
+ }
+}
+
void MacroAssembler::store_klass(Register dst, Register src) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
+ assert(!UseCompactObjectHeaders, "not with compact headers");
if (UseCompressedClassPointers) {
encode_klass_not_null(src);
strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
@@ -4751,6 +4796,7 @@ void MacroAssembler::store_klass(Register dst, Register src) {
}
void MacroAssembler::store_klass_gap(Register dst, Register src) {
+ assert(!UseCompactObjectHeaders, "not with compact headers");
if (UseCompressedClassPointers) {
// Store to klass gap in destination
strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
@@ -6573,97 +6619,122 @@ void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) {
}
// Implements lightweight-locking.
-// Branches to slow upon failure to lock the object, with ZF cleared.
-// Falls through upon success with ZF set.
//
// - obj: the object to be locked
-// - hdr: the header, already loaded from obj, will be destroyed
-// - t1, t2: temporary registers, will be destroyed
-void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
+// - t1, t2, t3: temporary registers, will be destroyed
+// - slow: branched to if locking fails, absolute offset may larger than 32KB (imm14 encoding).
+void MacroAssembler::lightweight_lock(Register obj, Register t1, Register t2, Register t3, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
- assert_different_registers(obj, hdr, t1, t2, rscratch1);
-
- // Check if we would have space on lock-stack for the object.
- ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
- cmpw(t1, (unsigned)LockStack::end_offset() - 1);
- br(Assembler::GT, slow);
-
- // Load (object->mark() | 1) into hdr
- orr(hdr, hdr, markWord::unlocked_value);
- // Clear lock-bits, into t2
- eor(t2, hdr, markWord::unlocked_value);
- // Try to swing header from unlocked to locked
- // Clobbers rscratch1 when UseLSE is false
- cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
- /*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
+ assert_different_registers(obj, t1, t2, t3, rscratch1);
+
+ Label push;
+ const Register top = t1;
+ const Register mark = t2;
+ const Register t = t3;
+
+ // Preload the markWord. It is important that this is the first
+ // instruction emitted as it is part of C1's null check semantics.
+ ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Check if the lock-stack is full.
+ ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ cmpw(top, (unsigned)LockStack::end_offset());
+ br(Assembler::GE, slow);
+
+ // Check for recursion.
+ subw(t, top, oopSize);
+ ldr(t, Address(rthread, t));
+ cmp(obj, t);
+ br(Assembler::EQ, push);
+
+ // Check header for monitor (0b10).
+ tst(mark, markWord::monitor_value);
br(Assembler::NE, slow);
- // After successful lock, push object on lock-stack
- ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
- str(obj, Address(rthread, t1));
- addw(t1, t1, oopSize);
- strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+ // Try to lock. Transition lock bits 0b01 => 0b00
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
+ orr(mark, mark, markWord::unlocked_value);
+ eor(t, mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::xword,
+ /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
+ br(Assembler::NE, slow);
+
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ str(obj, Address(rthread, top));
+ addw(top, top, oopSize);
+ strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
}
// Implements lightweight-unlocking.
-// Branches to slow upon failure, with ZF cleared.
-// Falls through upon success, with ZF set.
//
// - obj: the object to be unlocked
-// - hdr: the (pre-loaded) header of the object
-// - t1, t2: temporary registers
-void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
+// - t1, t2, t3: temporary registers
+// - slow: branched to if unlocking fails, absolute offset may larger than 32KB (imm14 encoding).
+void MacroAssembler::lightweight_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
- assert_different_registers(obj, hdr, t1, t2, rscratch1);
+ // cmpxchg clobbers rscratch1.
+ assert_different_registers(obj, t1, t2, t3, rscratch1);
#ifdef ASSERT
{
- // The following checks rely on the fact that LockStack is only ever modified by
- // its owning thread, even if the lock got inflated concurrently; removal of LockStack
- // entries after inflation will happen delayed in that case.
-
// Check for lock-stack underflow.
Label stack_ok;
ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
cmpw(t1, (unsigned)LockStack::start_offset());
- br(Assembler::GT, stack_ok);
+ br(Assembler::GE, stack_ok);
STOP("Lock-stack underflow");
bind(stack_ok);
}
- {
- // Check if the top of the lock-stack matches the unlocked object.
- Label tos_ok;
- subw(t1, t1, oopSize);
- ldr(t1, Address(rthread, t1));
- cmpoop(t1, obj);
- br(Assembler::EQ, tos_ok);
- STOP("Top of lock-stack does not match the unlocked object");
- bind(tos_ok);
- }
- {
- // Check that hdr is fast-locked.
- Label hdr_ok;
- tst(hdr, markWord::lock_mask_in_place);
- br(Assembler::EQ, hdr_ok);
- STOP("Header is not fast-locked");
- bind(hdr_ok);
- }
#endif
- // Load the new header (unlocked) into t1
- orr(t1, hdr, markWord::unlocked_value);
+ Label unlocked, push_and_slow;
+ const Register top = t1;
+ const Register mark = t2;
+ const Register t = t3;
- // Try to swing header from locked to unlocked
- // Clobbers rscratch1 when UseLSE is false
- cmpxchg(obj, hdr, t1, Assembler::xword,
- /*acquire*/ true, /*release*/ true, /*weak*/ false, t2);
+ // Check if obj is top of lock-stack.
+ ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ subw(top, top, oopSize);
+ ldr(t, Address(rthread, top));
+ cmp(obj, t);
br(Assembler::NE, slow);
- // After successful unlock, pop object from lock-stack
- ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
- subw(t1, t1, oopSize);
+ // Pop lock-stack.
+ DEBUG_ONLY(str(zr, Address(rthread, top));)
+ strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+
+ // Check if recursive.
+ subw(t, top, oopSize);
+ ldr(t, Address(rthread, t));
+ cmp(obj, t);
+ br(Assembler::EQ, unlocked);
+
+ // Not recursive. Check header for monitor (0b10).
+ ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ tbnz(mark, log2i_exact(markWord::monitor_value), push_and_slow);
+
#ifdef ASSERT
- str(zr, Address(rthread, t1));
+ // Check header not unlocked (0b01).
+ Label not_unlocked;
+ tbz(mark, log2i_exact(markWord::unlocked_value), not_unlocked);
+ stop("lightweight_unlock already unlocked");
+ bind(not_unlocked);
#endif
- strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
+ orr(t, mark, markWord::unlocked_value);
+ cmpxchg(obj, mark, t, Assembler::xword,
+ /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
+ br(Assembler::EQ, unlocked);
+
+ bind(push_and_slow);
+ // Restore lock-stack and handle the unlock in runtime.
+ DEBUG_ONLY(str(obj, Address(rthread, top));)
+ addw(top, top, oopSize);
+ strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ b(slow);
+
+ bind(unlocked);
}
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -862,9 +862,11 @@ public:
void load_method_holder(Register holder, Register method);
// oop manipulations
+ void load_nklass_compact(Register dst, Register src);
void load_klass(Register dst, Register src);
void store_klass(Register dst, Register src);
void cmp_klass(Register oop, Register trial_klass, Register tmp);
+ void cmp_klass(Register src, Register dst, Register tmp1, Register tmp2);
void resolve_weak_handle(Register result, Register tmp1, Register tmp2);
void resolve_oop_handle(Register result, Register tmp1, Register tmp2);
@@ -1648,8 +1650,8 @@ public:
// Code for java.lang.Thread::onSpinWait() intrinsic.
void spin_wait();
- void lightweight_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow);
- void lightweight_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow);
+ void lightweight_lock(Register obj, Register t1, Register t2, Register t3, Label& slow);
+ void lightweight_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow);
private:
// Check the current thread doesn't need a cross modify fence.
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1817,7 +1817,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ br(Assembler::NE, slow_path_lock);
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- __ ldr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
__ lightweight_lock(obj_reg, swap_reg, tmp, lock_tmp, slow_path_lock);
}
__ bind(count);
@@ -1960,8 +1959,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "");
- __ ldr(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- __ tbnz(old_hdr, exact_log2(markWord::monitor_value), slow_path_unlock);
__ lightweight_unlock(obj_reg, old_hdr, swap_reg, lock_tmp, slow_path_unlock);
__ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
}
@@ -3573,12 +3573,22 @@ void TemplateTable::_new() {
// The object is initialized before the header. If the object size is
// zero, go directly to the header initialization.
- __ sub(r3, r3, sizeof(oopDesc));
+ if (UseCompactObjectHeaders) {
+ assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
+ __ sub(r3, r3, oopDesc::base_offset_in_bytes());
+ } else {
+ __ sub(r3, r3, sizeof(oopDesc));
+ }
__ cbz(r3, initialize_header);
// Initialize object fields
{
- __ add(r2, r0, sizeof(oopDesc));
+ if (UseCompactObjectHeaders) {
+ assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
+ __ add(r2, r0, oopDesc::base_offset_in_bytes());
+ } else {
+ __ add(r2, r0, sizeof(oopDesc));
+ }
Label loop;
__ bind(loop);
__ str(zr, Address(__ post(r2, BytesPerLong)));
@@ -3588,11 +3598,15 @@ void TemplateTable::_new() {
// initialize object header only.
__ bind(initialize_header);
- __ mov(rscratch1, (intptr_t)markWord::prototype().value());
- __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
- __ store_klass_gap(r0, zr); // zero klass gap for compressed oops
- __ store_klass(r0, r4); // store klass last
-
+ if (UseCompactObjectHeaders) {
+ __ ldr(rscratch1, Address(r4, Klass::prototype_header_offset()));
+ __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
+ } else {
+ __ mov(rscratch1, (intptr_t)markWord::prototype().value());
+ __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
+ __ store_klass_gap(r0, zr); // zero klass gap for compressed oops
+ __ store_klass(r0, r4); // store klass last
+ }
{
SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
// Trigger dtrace event for fastpath
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -176,6 +176,7 @@ enum Ampere_CPU_Model {
static bool supports_fast_class_init_checks() { return true; }
constexpr static bool supports_stack_watermark_barrier() { return true; }
+ constexpr static bool supports_recursive_lightweight_locking() { return true; }
static void get_compatible_board(char *buf, int buflen);
@@ -217,6 +217,10 @@ void MonitorExitStub::emit_code(LIR_Assembler* ce) {
__ b(_continuation);
}
+void LoadKlassStub::emit_code(LIR_Assembler* ce) {
+ // Currently not needed.
+ Unimplemented();
+}
// Call return is directly after patch word
int PatchingStub::_patch_info_offset = 0;
@@ -971,7 +971,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
op->tmp1()->as_register(),
op->tmp2()->as_register(),
op->tmp3()->as_register(),
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
type2aelembytes(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -298,6 +298,10 @@ void MonitorExitStub::emit_code(LIR_Assembler* ce) {
__ b(_continuation);
}
+void LoadKlassStub::emit_code(LIR_Assembler* ce) {
+ // Currently not needed.
+ Unimplemented();
+}
// Implementation of patching:
// - Copy the code at given offset to an inlined buffer (first the bytes, then the number of bytes).
@@ -2328,7 +2328,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
op->tmp1()->as_register(),
op->tmp2()->as_register(),
op->tmp3()->as_register(),
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
type2aelembytes(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -337,7 +337,7 @@ void C1_MacroAssembler::allocate_array(
Register t1, // temp register
Register t2, // temp register
Register t3, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int elt_size, // element size in bytes
Register klass, // object klass
Label& slow_case // continuation point if fast allocation fails
@@ -369,7 +369,7 @@ void C1_MacroAssembler::allocate_array(
sldi(t1, len, log2_elt_size);
arr_len_in_bytes = t1;
}
- addi(arr_size, arr_len_in_bytes, hdr_size * wordSize + MinObjAlignmentInBytesMask); // Add space for header & alignment.
+ addi(arr_size, arr_len_in_bytes, base_offset_in_bytes + MinObjAlignmentInBytesMask); // Add space for header & alignment.
clrrdi(arr_size, arr_size, LogMinObjAlignmentInBytes); // Align array size.
// Allocate space & initialize header.
@@ -379,8 +379,18 @@ void C1_MacroAssembler::allocate_array(
// Initialize body.
const Register base = t2;
const Register index = t3;
- addi(base, obj, hdr_size * wordSize); // compute address of first element
- addi(index, arr_size, -(hdr_size * wordSize)); // compute index = number of bytes to clear
+ addi(base, obj, base_offset_in_bytes); // compute address of first element
+ addi(index, arr_size, -(base_offset_in_bytes)); // compute index = number of bytes to clear
+
+ // Zero first 4 bytes, if start offset is not word aligned.
+ if (!is_aligned(base_offset_in_bytes, BytesPerWord)) {
+ assert(is_aligned(base_offset_in_bytes, BytesPerInt), "must be 4-byte aligned");
+ li(t1, 0);
+ stw(t1, 0, base);
+ addi(base, base, BytesPerInt);
+ // Note: initialize_body will align index down, no need to correct it here.
+ }
+
initialize_body(base, index);
if (CURRENT_ENV->dtrace_alloc_probes()) {
@@ -80,7 +80,7 @@
Register t1, // temp register
Register t2, // temp register
Register t3, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int elt_size, // element size in bytes
Register klass, // object klass
Label& slow_case // continuation point if fast allocation fails
@@ -228,6 +228,11 @@ void MonitorExitStub::emit_code(LIR_Assembler* ce) {
__ far_jump(RuntimeAddress(Runtime1::entry_for(exit_id)));
}
+void LoadKlassStub::emit_code(LIR_Assembler* ce) {
+ // Currently not needed.
+ Unimplemented();
+}
+
// Implementation of patching:
// - Copy the code at given offset to an inlined buffer (first the bytes, then the number of bytes)
// - Replace original code with a call to the stub
@@ -1040,7 +1040,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
len,
tmp1,
tmp2,
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
array_element_size(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -181,6 +181,10 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (len->is_valid()) {
sw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
+ if (!is_aligned(arrayOopDesc::header_size_in_bytes(), BytesPerWord)) {
+ assert(is_aligned(arrayOopDesc::header_size_in_bytes(), BytesPerInt), "must be 4-byte aligned");
+ sw(zr, Address(obj, arrayOopDesc::header_size_in_bytes()));
+ }
} else if (UseCompressedClassPointers) {
store_klass_gap(obj, zr);
}
@@ -280,7 +284,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
verify_oop(obj);
}
-void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int header_size, int f, Register klass, Label& slow_case) {
+void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
assert_different_registers(obj, len, tmp1, tmp2, klass);
// determine alignment mask
@@ -292,7 +296,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1
const Register arr_size = tmp2; // okay to be the same
// align object end
- mv(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+ mv(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
shadd(arr_size, len, arr_size, t0, f);
andi(arr_size, arr_size, ~(uint)MinObjAlignmentInBytesMask);
@@ -302,7 +306,10 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1
// clear rest of allocated space
const Register len_zero = len;
- initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
+ // We align-up the header size to word-size, because we clear the
+ // possible alignment gap in initialize_header().
+ int hdr_size = align_up(base_offset_in_bytes, BytesPerWord);
+ initialize_body(obj, arr_size, hdr_size, len_zero);
membar(MacroAssembler::StoreStore);
@@ -101,7 +101,7 @@ using MacroAssembler::null_check;
// header_size: size of object header in words
// f : element scale factor
// slow_case : exit to slow case implementation if fast allocation fails
- void allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int header_size, int f, Register klass, Label& slow_case);
+ void allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int base_offset_in_bytes, int f, Register klass, Label& slow_case);
int rsp_offset() const { return _rsp_offset; }
@@ -253,6 +253,11 @@ void MonitorExitStub::emit_code(LIR_Assembler* ce) {
__ branch_optimized(Assembler::bcondAlways, _continuation);
}
+void LoadKlassStub::emit_code(LIR_Assembler* ce) {
+ // Currently not needed.
+ Unimplemented();
+}
+
// Implementation of patching:
// - Copy the code at given offset to an inlined buffer (first the bytes, then the number of bytes).
// - Replace original code with a call to the stub.
@@ -2403,7 +2403,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
op->len()->as_register(),
op->tmp1()->as_register(),
op->tmp2()->as_register(),
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
type2aelembytes(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -300,7 +300,7 @@ void C1_MacroAssembler::allocate_array(
Register len, // array length
Register t1, // temp register
Register t2, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int elt_size, // element size in bytes
Register klass, // object klass
Label& slow_case // Continuation point if fast allocation fails.
@@ -326,8 +326,8 @@ void C1_MacroAssembler::allocate_array(
case 8: z_sllg(arr_size, len, 3); break;
default: ShouldNotReachHere();
}
- add2reg(arr_size, hdr_size * wordSize + MinObjAlignmentInBytesMask); // Add space for header & alignment.
- z_nill(arr_size, (~MinObjAlignmentInBytesMask) & 0xffff); // Align array size.
+ add2reg(arr_size, base_offset_in_bytes + MinObjAlignmentInBytesMask); // Add space for header & alignment.
+ z_nill(arr_size, (~MinObjAlignmentInBytesMask) & 0xffff); // Align array size.
try_allocate(obj, arr_size, 0, t1, slow_case);
@@ -337,9 +337,9 @@ void C1_MacroAssembler::allocate_array(
Label done;
Register object_fields = t1;
Register Rzero = Z_R1_scratch;
- z_aghi(arr_size, -(hdr_size * BytesPerWord));
+ z_aghi(arr_size, -base_offset_in_bytes);
z_bre(done); // Jump if size of fields is zero.
- z_la(object_fields, hdr_size * BytesPerWord, obj);
+ z_la(object_fields, base_offset_in_bytes, obj);
z_xgr(Rzero, Rzero);
initialize_body(object_fields, arr_size, Rzero);
bind(done);
@@ -70,7 +70,7 @@
Register obj, // result: Pointer to object after successful allocation.
Register t1, // temp register
Register t2, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int obj_size, // object size in words
Register klass, // object klass
Label& slow_case // Continuation point if fast allocation fails.
@@ -1635,7 +1635,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
len,
tmp1,
tmp2,
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
array_element_size(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -178,6 +178,14 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (len->is_valid()) {
movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len);
+#ifdef _LP64
+ if (!is_aligned(arrayOopDesc::header_size_in_bytes(), BytesPerWord)) {
+ assert(is_aligned(arrayOopDesc::header_size_in_bytes(), BytesPerInt), "must be 4-byte aligned");
+ movl(Address(obj, arrayOopDesc::header_size_in_bytes()), 0);
+ }
+#else
+ assert(is_aligned(arrayOopDesc::header_size_in_bytes(), BytesPerInt), "must be 4-byte aligned");
+#endif
}
#ifdef _LP64
else if (UseCompressedClassPointers) {
@@ -261,7 +269,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
verify_oop(obj);
}
-void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, Address::ScaleFactor f, Register klass, Label& slow_case) {
+void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case) {
assert(obj == rax, "obj must be in rax, for cmpxchg");
assert_different_registers(obj, len, t1, t2, klass);
@@ -274,7 +282,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
const Register arr_size = t2; // okay to be the same
// align object end
- movptr(arr_size, header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+ movptr(arr_size, base_offset_in_bytes + MinObjAlignmentInBytesMask);
lea(arr_size, Address(arr_size, len, f));
andptr(arr_size, ~MinObjAlignmentInBytesMask);
@@ -284,7 +292,10 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
// clear rest of allocated space
const Register len_zero = len;
- initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
+ // We align-up the header size to word-size, because we clear the
+ // possible alignment gap in initialize_header().
+ int hdr_size = align_up(base_offset_in_bytes, BytesPerWord);
+ initialize_body(obj, arr_size, hdr_size, len_zero);
if (CURRENT_ENV->dtrace_alloc_probes()) {
assert(obj == rax, "must be");
@@ -76,7 +76,7 @@
// header_size: size of object header in words
// object_size: total size of object in words
// slow_case : exit to slow case implementation if fast allocation fails
- void allocate_object(Register obj, Register t1, Register t2, int header_size, int object_size, Register klass, Label& slow_case);
+ void allocate_object(Register obj, Register t1, Register t2, int base_offset_in_bytes, int object_size, Register klass, Label& slow_case);
enum {
max_array_allocation_length = 0x00FFFFFF
@@ -93,6 +93,18 @@ void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
#endif
__ jmp(continuation());
}
+
+int C2LoadNKlassStub::max_size() const {
+ return 10;
+}
+
+void C2LoadNKlassStub::emit(C2_MacroAssembler& masm) {
+ __ bind(entry());
+ Register d = dst();
+ __ movq(d, Address(d, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
+ __ jmp(continuation());
+}
+
#endif
#undef __
@@ -581,4 +581,22 @@ class ArrayCopyStub: public CodeStub {
#endif // PRODUCT
};
+class LoadKlassStub: public CodeStub {
+private:
+ LIR_Opr _result;
+
+public:
+ LoadKlassStub(LIR_Opr result) :
+ CodeStub(), _result(result) {};
+
+ virtual void emit_code(LIR_Assembler* e);
+ virtual void visit(LIR_OpVisitState* visitor) {
+ visitor->do_temp(_result);
+ visitor->do_output(_result);
+ }
+#ifndef PRODUCT
+ virtual void print_name(outputStream* out) const { out->print("LoadKlassStub"); }
+#endif // PRODUCT
+};
+
#endif // SHARE_C1_C1_CODESTUBS_HPP
@@ -890,6 +890,7 @@ void LIR_OpVisitState::visit(LIR_Op* op) {
do_input(opLoadKlass->_obj);
do_output(opLoadKlass->_result);
+ if (opLoadKlass->_stub) do_stub(opLoadKlass->_stub);
if (opLoadKlass->_info) do_info(opLoadKlass->_info);
break;
}
@@ -1070,6 +1071,9 @@ void LIR_OpLock::emit_code(LIR_Assembler* masm) {
void LIR_OpLoadKlass::emit_code(LIR_Assembler* masm) {
masm->emit_load_klass(this);
+ if (stub()) {
+ masm->append_code_stub(stub());
+ }
}
#ifdef ASSERT
@@ -2046,6 +2050,9 @@ void LIR_OpLock::print_instr(outputStream* out) const {
void LIR_OpLoadKlass::print_instr(outputStream* out) const {
obj()->print(out); out->print(" ");
result_opr()->print(out); out->print(" ");
+ if (stub()) {
+ out->print("[lbl:" INTPTR_FORMAT "]", p2i(stub()->entry()));
+ }
}
#ifdef ASSERT
@@ -1903,13 +1903,16 @@ class LIR_OpLoadKlass: public LIR_Op {
private:
LIR_Opr _obj;
+ CodeStub* _stub;
public:
- LIR_OpLoadKlass(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info)
+ LIR_OpLoadKlass(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, CodeStub* stub)
: LIR_Op(lir_load_klass, result, info)
, _obj(obj)
+ , _stub(stub)
{}
LIR_Opr obj() const { return _obj; }
+ CodeStub* stub() const { return _stub; }
virtual LIR_OpLoadKlass* as_OpLoadKlass() { return this; }
virtual void emit_code(LIR_Assembler* masm);
@@ -2375,7 +2378,7 @@ class LIR_List: public CompilationResourceObj {
void xadd(LIR_Opr src, LIR_Opr add, LIR_Opr res, LIR_Opr tmp) { append(new LIR_Op2(lir_xadd, src, add, res, tmp)); }
void xchg(LIR_Opr src, LIR_Opr set, LIR_Opr res, LIR_Opr tmp) { append(new LIR_Op2(lir_xchg, src, set, res, tmp)); }
- void load_klass(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info) { append(new LIR_OpLoadKlass(obj, result, info)); }
+ void load_klass(LIR_Opr obj, LIR_Opr result, CodeEmitInfo* info, CodeStub* stub) { append(new LIR_OpLoadKlass(obj, result, info, stub)); }
#ifdef ASSERT
void lir_assert(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, const char* msg, bool halt) { append(new LIR_OpAssert(condition, opr1, opr2, msg, halt)); }
@@ -1243,7 +1243,8 @@ void LIRGenerator::do_isInstance(Intrinsic* x) {
}
void LIRGenerator::load_klass(LIR_Opr obj, LIR_Opr klass, CodeEmitInfo* null_check_info) {
- __ load_klass(obj, klass, null_check_info);
+ CodeStub* slow_path = AARCH64_ONLY(UseCompactObjectHeaders ? new LoadKlassStub(klass) :) nullptr;
+ __ load_klass(obj, klass, null_check_info, slow_path);
}
// Example: object.getClass ()
@@ -686,6 +686,13 @@ void ArchiveBuilder::make_klasses_shareable() {
const char* generated = "";
Klass* k = get_buffered_addr(klasses()->at(i));
k->remove_java_mirror();
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ Klass* requested_k = to_requested(k);
+ narrowKlass nk = CompressedKlassPointers::encode_not_null(requested_k, _requested_static_archive_bottom);
+ k->set_prototype_header(markWord::prototype().set_narrow_klass(nk));
+ }
+#endif //AARCH64
if (k->is_objArray_klass()) {
// InstanceKlass and TypeArrayKlass will in turn call remove_unshareable_info
// on their array classes.
@@ -193,8 +193,15 @@ void ArchiveHeapWriter::copy_roots_to_buffer(GrowableArrayCHeap<oop, mtClassShar
memset(mem, 0, byte_size);
{
// This is copied from MemAllocator::finish
- oopDesc::set_mark(mem, markWord::prototype());
- oopDesc::release_set_klass(mem, k);
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ oopDesc::release_set_mark(mem, k->prototype_header());
+ } else
+#endif
+ {
+ oopDesc::set_mark(mem, markWord::prototype());
+ oopDesc::release_set_klass(mem, k);
+ }
}
{
// This is copied from ObjArrayAllocator::initialize
@@ -260,9 +267,16 @@ void ArchiveHeapWriter::init_filler_array_at_buffer_top(int array_length, size_t
Klass* oak = Universe::objectArrayKlassObj(); // already relocated to point to archived klass
HeapWord* mem = offset_to_buffered_address<HeapWord*>(_buffer_used);
memset(mem, 0, fill_bytes);
- oopDesc::set_mark(mem, markWord::prototype());
narrowKlass nk = ArchiveBuilder::current()->get_requested_narrow_klass(oak);
- cast_to_oop(mem)->set_narrow_klass(nk);
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ oopDesc::release_set_mark(mem, markWord::prototype().set_narrow_klass(nk));
+ } else
+#endif
+ {
+ oopDesc::set_mark(mem, markWord::prototype());
+ cast_to_oop(mem)->set_narrow_klass(nk);
+ }
arrayOopDesc::set_length(mem, array_length);
}
@@ -422,13 +436,27 @@ void ArchiveHeapWriter::update_header_for_requested_obj(oop requested_obj, oop s
address buffered_addr = requested_addr_to_buffered_addr(cast_from_oop<address>(requested_obj));
oop fake_oop = cast_to_oop(buffered_addr);
- fake_oop->set_narrow_klass(nk);
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ fake_oop->set_mark(fake_oop->mark().set_narrow_klass(nk));
+ } else
+#endif
+ {
+ fake_oop->set_narrow_klass(nk);
+ }
// We need to retain the identity_hash, because it may have been used by some hashtables
// in the shared heap.
if (src_obj != nullptr && !src_obj->fast_no_hash_check()) {
int src_hash = src_obj->identity_hash();
- fake_oop->set_mark(markWord::prototype().copy_set_hash(src_hash));
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ fake_oop->set_mark(markWord::prototype().set_narrow_klass(nk).copy_set_hash(src_hash));
+ } else
+#endif
+ {
+ fake_oop->set_mark(markWord::prototype().copy_set_hash(src_hash));
+ }
assert(fake_oop->mark().is_unlocked(), "sanity");
DEBUG_ONLY(int archived_hash = fake_oop->identity_hash());
@@ -203,6 +203,7 @@ void FileMapHeader::populate(FileMapInfo *info, size_t core_region_alignment,
_core_region_alignment = core_region_alignment;
_obj_alignment = ObjectAlignmentInBytes;
_compact_strings = CompactStrings;
+ _compact_headers = AARCH64_ONLY(UseCompactObjectHeaders) NOT_AARCH64(false);
if (DumpSharedSpaces && HeapShared::can_write()) {
_narrow_oop_mode = CompressedOops::mode();
_narrow_oop_base = CompressedOops::base();
@@ -281,6 +282,7 @@ void FileMapHeader::print(outputStream* st) {
st->print_cr("- narrow_oop_base: " INTPTR_FORMAT, p2i(_narrow_oop_base));
st->print_cr("- narrow_oop_shift %d", _narrow_oop_shift);
st->print_cr("- compact_strings: %d", _compact_strings);
+ st->print_cr("- compact_headers: %d", _compact_headers);
st->print_cr("- max_heap_size: " UINTX_FORMAT, _max_heap_size);
st->print_cr("- narrow_oop_mode: %d", _narrow_oop_mode);
st->print_cr("- narrow_klass_shift: %d", _narrow_klass_shift);
@@ -2419,6 +2421,14 @@ bool FileMapHeader::validate() {
return false;
}
+ if (compact_headers() != AARCH64_ONLY(UseCompactObjectHeaders) NOT_AARCH64(false)) {
+ log_info(cds)("The shared archive file's UseCompactObjectHeaders setting (%s)"
+ " does not equal the current UseCompactObjectHeaders setting (%s).",
+ _compact_headers ? "enabled" : "disabled",
+ AARCH64_ONLY(UseCompactObjectHeaders ? "enabled" : "disabled") NOT_AARCH64("disabled"));
+ return false;
+ }
+
if (!_use_optimized_module_handling) {
MetaspaceShared::disable_optimized_module_handling();
log_info(cds)("optimized module handling: disabled because archive was created without optimized module handling");
@@ -187,6 +187,7 @@ private:
address _narrow_oop_base; // compressed oop encoding base
int _narrow_oop_shift; // compressed oop encoding shift
bool _compact_strings; // value of CompactStrings
+ bool _compact_headers; // value of UseCompactObjectHeaders
uintx _max_heap_size; // java max heap size during dumping
CompressedOops::Mode _narrow_oop_mode; // compressed oop encoding mode
int _narrow_klass_shift; // save narrow klass base and shift
@@ -256,6 +257,7 @@ public:
address narrow_oop_base() const { return _narrow_oop_base; }
int narrow_oop_shift() const { return _narrow_oop_shift; }
bool compact_strings() const { return _compact_strings; }
+ bool compact_headers() const { return _compact_headers; }
uintx max_heap_size() const { return _max_heap_size; }
CompressedOops::Mode narrow_oop_mode() const { return _narrow_oop_mode; }
int narrow_klass_shift() const { return _narrow_klass_shift; }
@@ -249,3 +249,25 @@ const char* ciKlass::external_name() const {
return get_Klass()->external_name();
)
}
+
+// ------------------------------------------------------------------
+// ciKlass::prototype_header_offset
+juint ciKlass::prototype_header_offset() {
+ assert(is_loaded(), "must be loaded");
+
+ VM_ENTRY_MARK;
+ Klass* this_klass = get_Klass();
+ return in_bytes(this_klass->prototype_header_offset());
+}
+
+// ------------------------------------------------------------------
+// ciKlass::prototype_header
+#ifdef AARCH64
+uintptr_t ciKlass::prototype_header() {
+ assert(is_loaded(), "must be loaded");
+
+ VM_ENTRY_MARK;
+ Klass* this_klass = get_Klass();
+ return (uintptr_t)this_klass->prototype_header().to_pointer();
+}
+#endif
@@ -129,6 +129,11 @@ public:
void print_name_on(outputStream* st);
const char* external_name() const;
+
+ juint prototype_header_offset();
+#ifdef AARCH64
+ uintptr_t prototype_header();
+#endif
};
#endif // SHARE_CI_CIKLASS_HPP
@@ -88,6 +88,7 @@
#include "gc/shared/oopStorageParState.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/referenceProcessor.inline.hpp"
+#include "gc/shared/slidingForwarding.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shared/taskqueue.inline.hpp"
#include "gc/shared/taskTerminator.hpp"
@@ -1530,6 +1531,8 @@ jint G1CollectedHeap::initialize() {
G1InitLogger::print();
+ SlidingForwarding::initialize(heap_rs.region(), HeapRegion::GrainWords);
+
return JNI_OK;
}
@@ -40,6 +40,7 @@
#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shared/referenceProcessor.hpp"
+#include "gc/shared/slidingForwarding.hpp"
#include "gc/shared/verifyOption.hpp"
#include "gc/shared/weakProcessor.inline.hpp"
#include "gc/shared/workerPolicy.hpp"
@@ -209,6 +210,8 @@ void G1FullCollector::collect() {
// Don't add any more derived pointers during later phases
deactivate_derived_pointers();
+ SlidingForwarding::begin();
+
phase2_prepare_compaction();
if (has_compaction_targets()) {
@@ -221,6 +224,8 @@ void G1FullCollector::collect() {
log_info(gc, phases) ("No Regions selected for compaction. Skipping Phase 3: Adjust pointers and Phase 4: Compact heap");
}
+ SlidingForwarding::end();
+
phase5_reset_metadata();
G1CollectedHeap::finish_codecache_marking_cycle();
@@ -389,7 +394,8 @@ uint G1FullCollector::truncate_parallel_cps() {
return lowest_current;
}
-void G1FullCollector::phase2c_prepare_serial_compaction() {
+template <bool ALT_FWD>
+void G1FullCollector::phase2c_prepare_serial_compaction_impl() {
GCTraceTime(Debug, gc, phases) debug("Phase 2: Prepare serial compaction", scope()->timer());
// At this point, we know that after parallel compaction there will be regions that
// are partially compacted into. Thus, the last compaction region of all
@@ -414,7 +420,7 @@ void G1FullCollector::phase2c_prepare_serial_compaction() {
serial_cp->initialize(start_hr);
HeapWord* dense_prefix_top = compaction_top(start_hr);
- G1SerialRePrepareClosure re_prepare(serial_cp, dense_prefix_top);
+ G1SerialRePrepareClosure<ALT_FWD> re_prepare(serial_cp, dense_prefix_top);
for (uint i = start_serial + 1; i < _heap->max_reserved_regions(); i++) {
if (is_compaction_target(i)) {
@@ -427,7 +433,16 @@ void G1FullCollector::phase2c_prepare_serial_compaction() {
serial_cp->update();
}
-void G1FullCollector::phase2d_prepare_humongous_compaction() {
+void G1FullCollector::phase2c_prepare_serial_compaction() {
+ if (UseAltGCForwarding) {
+ phase2c_prepare_serial_compaction_impl<true>();
+ } else {
+ phase2c_prepare_serial_compaction_impl<false>();
+ }
+}
+
+template <bool ALT_FWD>
+void G1FullCollector::phase2d_prepare_humongous_compaction_impl() {
GCTraceTime(Debug, gc, phases) debug("Phase 2: Prepare humongous compaction", scope()->timer());
G1FullGCCompactionPoint* serial_cp = serial_compaction_point();
assert(serial_cp->has_regions(), "Sanity!" );
@@ -445,7 +460,7 @@ void G1FullCollector::phase2d_prepare_humongous_compaction() {
region_index++;
continue;
} else if (hr->is_starts_humongous()) {
- uint num_regions = humongous_cp->forward_humongous(hr);
+ uint num_regions = humongous_cp->forward_humongous<ALT_FWD>(hr);
region_index += num_regions; // Skip over the continues humongous regions.
continue;
} else if (is_compaction_target(region_index)) {
@@ -456,6 +471,14 @@ void G1FullCollector::phase2d_prepare_humongous_compaction() {
}
}
+void G1FullCollector::phase2d_prepare_humongous_compaction() {
+ if (UseAltGCForwarding) {
+ phase2d_prepare_humongous_compaction_impl<true>();
+ } else {
+ phase2d_prepare_humongous_compaction_impl<false>();
+ }
+}
+
void G1FullCollector::phase3_adjust_pointers() {
// Adjust the pointers to reflect the new locations
GCTraceTime(Info, gc, phases) info("Phase 3: Adjust pointers", scope()->timer());
@@ -158,7 +158,11 @@ private:
void phase2a_determine_worklists();
bool phase2b_forward_oops();
+ template <bool ALT_FWD>
+ void phase2c_prepare_serial_compaction_impl();
void phase2c_prepare_serial_compaction();
+ template <bool ALT_FWD>
+ void phase2d_prepare_humongous_compaction_impl();
void phase2d_prepare_humongous_compaction();
void phase3_adjust_pointers();
@@ -40,10 +40,11 @@
#include "memory/iterator.inline.hpp"
#include "runtime/atomic.hpp"
+template <bool ALT_FWD>
class G1AdjustLiveClosure : public StackObj {
- G1AdjustClosure* _adjust_closure;
+ G1AdjustClosure<ALT_FWD>* _adjust_closure;
public:
- G1AdjustLiveClosure(G1AdjustClosure* cl) :
+ G1AdjustLiveClosure(G1AdjustClosure<ALT_FWD>* cl) :
_adjust_closure(cl) { }
size_t apply(oop object) {
@@ -62,7 +63,17 @@ class G1AdjustRegionClosure : public HeapRegionClosure {
_worker_id(worker_id) { }
bool do_heap_region(HeapRegion* r) {
- G1AdjustClosure cl(_collector);
+ if (UseAltGCForwarding) {
+ return do_heap_region_impl<true>(r);
+ } else {
+ return do_heap_region_impl<false>(r);
+ }
+ }
+
+private:
+ template <bool ALT_FWD>
+ bool do_heap_region_impl(HeapRegion* r) {
+ G1AdjustClosure<ALT_FWD> cl(_collector);
if (r->is_humongous()) {
// Special handling for humongous regions to get somewhat better
// work distribution.
@@ -70,7 +81,7 @@ class G1AdjustRegionClosure : public HeapRegionClosure {
obj->oop_iterate(&cl, MemRegion(r->bottom(), r->top()));
} else if (!r->is_free()) {
// Free regions do not contain objects to iterate. So skip them.
- G1AdjustLiveClosure adjust(&cl);
+ G1AdjustLiveClosure<ALT_FWD> adjust(&cl);
r->apply_to_marked_objects(_bitmap, &adjust);
}
return false;
@@ -81,12 +92,12 @@ G1FullGCAdjustTask::G1FullGCAdjustTask(G1FullCollector* collector) :
G1FullGCTask("G1 Adjust", collector),
_root_processor(G1CollectedHeap::heap(), collector->workers()),
_weak_proc_task(collector->workers()),
- _hrclaimer(collector->workers()),
- _adjust(collector) {
+ _hrclaimer(collector->workers()) {
ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
}
-void G1FullGCAdjustTask::work(uint worker_id) {
+template <bool ALT_FWD>
+void G1FullGCAdjustTask::work_impl(uint worker_id) {
Ticks start = Ticks::now();
ResourceMark rm;
@@ -94,18 +105,27 @@ void G1FullGCAdjustTask::work(uint worker_id) {
G1FullGCMarker* marker = collector()->marker(worker_id);
marker->preserved_stack()->adjust_during_full_gc();
+ G1AdjustClosure<ALT_FWD> adjust(collector());
{
// Adjust the weak roots.
AlwaysTrueClosure always_alive;
- _weak_proc_task.work(worker_id, &always_alive, &_adjust);
+ _weak_proc_task.work(worker_id, &always_alive, &adjust);
}
- CLDToOopClosure adjust_cld(&_adjust, ClassLoaderData::_claim_stw_fullgc_adjust);
- CodeBlobToOopClosure adjust_code(&_adjust, CodeBlobToOopClosure::FixRelocations);
- _root_processor.process_all_roots(&_adjust, &adjust_cld, &adjust_code);
+ CLDToOopClosure adjust_cld(&adjust, ClassLoaderData::_claim_stw_fullgc_adjust);
+ CodeBlobToOopClosure adjust_code(&adjust, CodeBlobToOopClosure::FixRelocations);
+ _root_processor.process_all_roots(&adjust, &adjust_cld, &adjust_code);
// Now adjust pointers region by region
G1AdjustRegionClosure blk(collector(), worker_id);
G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&blk, &_hrclaimer, worker_id);
log_task("Adjust task", worker_id, start);
}
+
+void G1FullGCAdjustTask::work(uint worker_id) {
+ if (UseAltGCForwarding) {
+ work_impl<true>(worker_id);
+ } else {
+ work_impl<false>(worker_id);
+ }
+}
@@ -38,8 +38,9 @@ class G1FullGCAdjustTask : public G1FullGCTask {
G1RootProcessor _root_processor;
WeakProcessor::Task _weak_proc_task;
HeapRegionClaimer _hrclaimer;
- G1AdjustClosure _adjust;
+ template <bool ALT_FWD>
+ void work_impl(uint worker_id);
public:
G1FullGCAdjustTask(G1FullCollector* collector);
void work(uint worker_id);
@@ -30,19 +30,22 @@
#include "gc/g1/g1FullGCCompactTask.hpp"
#include "gc/g1/heapRegion.inline.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
#include "logging/log.hpp"
#include "oops/oop.inline.hpp"
#include "utilities/ticks.hpp"
-void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) {
+template <bool ALT_FWD>
+void G1FullGCCompactTask::G1CompactRegionClosure<ALT_FWD>::clear_in_bitmap(oop obj) {
assert(_bitmap->is_marked(obj), "Should only compact marked objects");
_bitmap->clear(obj);
}
-size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
+template <bool ALT_FWD>
+size_t G1FullGCCompactTask::G1CompactRegionClosure<ALT_FWD>::apply(oop obj) {
size_t size = obj->size();
- if (obj->is_forwarded()) {
- G1FullGCCompactTask::copy_object_to_new_location(obj);
+ if (SlidingForwarding::is_forwarded(obj)) {
+ G1FullGCCompactTask::copy_object_to_new_location<ALT_FWD>(obj);
}
// Clear the mark for the compacted object to allow reuse of the
@@ -51,14 +54,15 @@ size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
return size;
}
+template <bool ALT_FWD>
void G1FullGCCompactTask::copy_object_to_new_location(oop obj) {
- assert(obj->is_forwarded(), "Sanity!");
- assert(obj->forwardee() != obj, "Object must have a new location");
+ assert(SlidingForwarding::is_forwarded(obj), "Sanity!");
+ assert(SlidingForwarding::forwardee<ALT_FWD>(obj) != obj, "Object must have a new location");
size_t size = obj->size();
// Copy object and reinit its mark.
HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
- HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
+ HeapWord* destination = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<ALT_FWD>(obj));
Copy::aligned_conjoint_words(obj_addr, destination, size);
// There is no need to transform stack chunks - marking already did that.
@@ -77,8 +81,13 @@ void G1FullGCCompactTask::compact_region(HeapRegion* hr) {
// showed that it was better overall to clear bit by bit, compared
// to clearing the whole region at the end. This difference was
// clearly seen for regions with few marks.
- G1CompactRegionClosure compact(collector()->mark_bitmap());
- hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
+ if (UseAltGCForwarding) {
+ G1CompactRegionClosure<true> compact(collector()->mark_bitmap());
+ hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
+ } else {
+ G1CompactRegionClosure<false> compact(collector()->mark_bitmap());
+ hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
+ }
}
hr->reset_compacted_after_full_gc(_collector->compaction_top(hr));
@@ -104,15 +113,24 @@ void G1FullGCCompactTask::serial_compaction() {
}
}
-void G1FullGCCompactTask::humongous_compaction() {
- GCTraceTime(Debug, gc, phases) tm("Phase 4: Humonguous Compaction", collector()->scope()->timer());
-
+template <bool ALT_FWD>
+void G1FullGCCompactTask::humongous_compaction_impl() {
for (HeapRegion* hr : collector()->humongous_compaction_regions()) {
assert(collector()->is_compaction_target(hr->hrm_index()), "Sanity");
- compact_humongous_obj(hr);
+ compact_humongous_obj<ALT_FWD>(hr);
+ }
+}
+
+void G1FullGCCompactTask::humongous_compaction() {
+ GCTraceTime(Debug, gc, phases) tm("Phase 4: Humonguous Compaction", collector()->scope()->timer());
+ if (UseAltGCForwarding) {
+ humongous_compaction_impl<true>();
+ } else {
+ humongous_compaction_impl<false>();
}
}
+template <bool ALT_FWD>
void G1FullGCCompactTask::compact_humongous_obj(HeapRegion* src_hr) {
assert(src_hr->is_starts_humongous(), "Should be start region of the humongous object");
@@ -120,12 +138,12 @@ void G1FullGCCompactTask::compact_humongous_obj(HeapRegion* src_hr) {
size_t word_size = obj->size();
uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(word_size);
- HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
+ HeapWord* destination = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<ALT_FWD>(obj));
assert(collector()->mark_bitmap()->is_marked(obj), "Should only compact marked objects");
collector()->mark_bitmap()->clear(obj);
- copy_object_to_new_location(obj);
+ copy_object_to_new_location<ALT_FWD>(obj);
uint dest_start_idx = _g1h->addr_to_region(destination);
// Update the metadata for the destination regions.
@@ -41,11 +41,16 @@ class G1FullGCCompactTask : public G1FullGCTask {
G1CollectedHeap* _g1h;
void compact_region(HeapRegion* hr);
+ template <bool ALT_FWD>
void compact_humongous_obj(HeapRegion* hr);
void free_non_overlapping_regions(uint src_start_idx, uint dest_start_idx, uint num_regions);
+ template <bool ALT_FWD>
static void copy_object_to_new_location(oop obj);
+ template <bool ALT_FWD>
+ void humongous_compaction_impl();
+
public:
G1FullGCCompactTask(G1FullCollector* collector) :
G1FullGCTask("G1 Compact Task", collector),
@@ -57,6 +62,7 @@ public:
void serial_compaction();
void humongous_compaction();
+ template <bool ALT_FWD>
class G1CompactRegionClosure : public StackObj {
G1CMBitMap* _bitmap;
void clear_in_bitmap(oop object);
@@ -27,6 +27,7 @@
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/heapRegion.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
#include "oops/oop.inline.hpp"
#include "utilities/debug.hpp"
@@ -92,6 +93,7 @@ void G1FullGCCompactionPoint::switch_region() {
initialize_values();
}
+template <bool ALT_FWD>
void G1FullGCCompactionPoint::forward(oop object, size_t size) {
assert(_current_region != nullptr, "Must have been initialized");
@@ -102,10 +104,10 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) {
// Store a forwarding pointer if the object should be moved.
if (cast_from_oop<HeapWord*>(object) != _compaction_top) {
- object->forward_to(cast_to_oop(_compaction_top));
- assert(object->is_forwarded(), "must be forwarded");
+ SlidingForwarding::forward_to<ALT_FWD>(object, cast_to_oop(_compaction_top));
+ assert(SlidingForwarding::is_forwarded(object), "must be forwarded");
} else {
- assert(!object->is_forwarded(), "must not be forwarded");
+ assert(SlidingForwarding::is_not_forwarded(object), "must not be forwarded");
}
// Update compaction values.
@@ -113,6 +115,9 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) {
_current_region->update_bot_for_block(_compaction_top - size, _compaction_top);
}
+template void G1FullGCCompactionPoint::forward<true>(oop object, size_t size);
+template void G1FullGCCompactionPoint::forward<false>(oop object, size_t size);
+
void G1FullGCCompactionPoint::add(HeapRegion* hr) {
_compaction_regions->append(hr);
}
@@ -145,6 +150,7 @@ void G1FullGCCompactionPoint::add_humongous(HeapRegion* hr) {
});
}
+template <bool ALT_FWD>
uint G1FullGCCompactionPoint::forward_humongous(HeapRegion* hr) {
assert(hr->is_starts_humongous(), "Sanity!");
@@ -168,8 +174,8 @@ uint G1FullGCCompactionPoint::forward_humongous(HeapRegion* hr) {
_collector->marker(0)->preserved_stack()->push_if_necessary(obj, obj->mark());
HeapRegion* dest_hr = _compaction_regions->at(range_begin);
- obj->forward_to(cast_to_oop(dest_hr->bottom()));
- assert(obj->is_forwarded(), "Object must be forwarded!");
+ SlidingForwarding::forward_to<ALT_FWD>(obj, cast_to_oop(dest_hr->bottom()));
+ assert(SlidingForwarding::is_forwarded(obj), "Object must be forwarded!");
// Add the humongous object regions to the compaction point.
add_humongous(hr);
@@ -180,6 +186,9 @@ uint G1FullGCCompactionPoint::forward_humongous(HeapRegion* hr) {
return num_regions;
}
+template uint G1FullGCCompactionPoint::forward_humongous<true>(HeapRegion* hr);
+template uint G1FullGCCompactionPoint::forward_humongous<false>(HeapRegion* hr);
+
uint G1FullGCCompactionPoint::find_contiguous_before(HeapRegion* hr, uint num_regions) {
assert(num_regions > 0, "Sanity!");
assert(has_regions(), "Sanity!");
@@ -54,7 +54,9 @@ public:
bool is_initialized();
void initialize(HeapRegion* hr);
void update();
+ template <bool ALT_FWD>
void forward(oop object, size_t size);
+ template <bool ALT_FWD>
uint forward_humongous(HeapRegion* hr);
void add(HeapRegion* hr);
void add_humongous(HeapRegion* hr);
@@ -73,6 +73,7 @@ public:
virtual void do_oop(narrowOop* p);
};
+template <bool ALT_FWD>
class G1AdjustClosure : public BasicOopIterateClosure {
G1FullCollector* _collector;
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
#include "gc/g1/g1FullGCMarker.inline.hpp"
#include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
#include "memory/iterator.inline.hpp"
#include "memory/universe.hpp"
#include "oops/access.inline.hpp"
@@ -51,7 +52,8 @@ inline void G1MarkAndPushClosure::do_oop(narrowOop* p) {
do_oop_work(p);
}
-template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
+template <bool ALT_FWD>
+template <class T> inline void G1AdjustClosure<ALT_FWD>::adjust_pointer(T* p) {
T heap_oop = RawAccess<>::oop_load(p);
if (CompressedOops::is_null(heap_oop)) {
return;
@@ -65,8 +67,8 @@ template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
return;
}
- if (obj->is_forwarded()) {
- oop forwardee = obj->forwardee();
+ if (SlidingForwarding::is_forwarded(obj)) {
+ oop forwardee = SlidingForwarding::forwardee<ALT_FWD>(obj);
// Forwarded, just update.
assert(G1CollectedHeap::heap()->is_in_reserved(forwardee), "should be in object space");
RawAccess<IS_NOT_NULL>::oop_store(p, forwardee);
@@ -74,8 +76,10 @@ template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
}
-inline void G1AdjustClosure::do_oop(oop* p) { do_oop_work(p); }
-inline void G1AdjustClosure::do_oop(narrowOop* p) { do_oop_work(p); }
+template <bool ALT_FWD>
+inline void G1AdjustClosure<ALT_FWD>::do_oop(oop* p) { do_oop_work(p); }
+template <bool ALT_FWD>
+inline void G1AdjustClosure<ALT_FWD>::do_oop(narrowOop* p) { do_oop_work(p); }
inline bool G1IsAliveClosure::do_object_b(oop p) {
return _bitmap->is_marked(p);
@@ -104,18 +104,25 @@ G1FullGCPrepareTask::G1CalculatePointersClosure::G1CalculatePointersClosure(G1Fu
_cp(cp) { }
-G1FullGCPrepareTask::G1PrepareCompactLiveClosure::G1PrepareCompactLiveClosure(G1FullGCCompactionPoint* cp) :
+template <bool ALT_FWD>
+G1FullGCPrepareTask::G1PrepareCompactLiveClosure<ALT_FWD>::G1PrepareCompactLiveClosure(G1FullGCCompactionPoint* cp) :
_cp(cp) { }
-size_t G1FullGCPrepareTask::G1PrepareCompactLiveClosure::apply(oop object) {
+template <bool ALT_FWD>
+size_t G1FullGCPrepareTask::G1PrepareCompactLiveClosure<ALT_FWD>::apply(oop object) {
size_t size = object->size();
- _cp->forward(object, size);
+ _cp->forward<ALT_FWD>(object, size);
return size;
}
void G1FullGCPrepareTask::G1CalculatePointersClosure::prepare_for_compaction(HeapRegion* hr) {
if (!_collector->is_free(hr->hrm_index())) {
- G1PrepareCompactLiveClosure prepare_compact(_cp);
- hr->apply_to_marked_objects(_bitmap, &prepare_compact);
+ if (UseAltGCForwarding) {
+ G1PrepareCompactLiveClosure<true> prepare_compact(_cp);
+ hr->apply_to_marked_objects(_bitmap, &prepare_compact);
+ } else {
+ G1PrepareCompactLiveClosure<false> prepare_compact(_cp);
+ hr->apply_to_marked_objects(_bitmap, &prepare_compact);
+ }
}
}
@@ -89,6 +89,7 @@ private:
bool do_heap_region(HeapRegion* hr);
};
+ template <bool ALT_FWD>
class G1PrepareCompactLiveClosure : public StackObj {
G1FullGCCompactionPoint* _cp;
@@ -100,6 +101,7 @@ private:
// Closure to re-prepare objects in the serial compaction point queue regions for
// serial compaction.
+template <bool ALT_FWD>
class G1SerialRePrepareClosure : public StackObj {
G1FullGCCompactionPoint* _cp;
HeapWord* _dense_prefix_top;
@@ -32,6 +32,7 @@
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCScope.hpp"
#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
void G1DetermineCompactionQueueClosure::free_empty_humongous_region(HeapRegion* hr) {
_g1h->free_humongous_region(hr, nullptr);
@@ -101,18 +102,19 @@ inline bool G1DetermineCompactionQueueClosure::do_heap_region(HeapRegion* hr) {
return false;
}
-inline size_t G1SerialRePrepareClosure::apply(oop obj) {
- if (obj->is_forwarded()) {
+template <bool ALT_FWD>
+inline size_t G1SerialRePrepareClosure<ALT_FWD>::apply(oop obj) {
+ if (SlidingForwarding::is_forwarded(obj)) {
// We skip objects compiled into the first region or
// into regions not part of the serial compaction point.
- if (cast_from_oop<HeapWord*>(obj->forwardee()) < _dense_prefix_top) {
+ if (cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<ALT_FWD>(obj)) < _dense_prefix_top) {
return obj->size();
}
}
// Get size and forward.
size_t size = obj->size();
- _cp->forward(obj, size);
+ _cp->forward<ALT_FWD>(obj, size);
return size;
}
@@ -228,7 +228,7 @@ void G1ParCopyClosure<barrier, should_mark>::do_oop_work(T* p) {
oop forwardee;
markWord m = obj->mark();
if (m.is_marked()) {
- forwardee = cast_to_oop(m.decode_pointer());
+ forwardee = obj->forwardee(m);
} else {
forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m);
}
@@ -207,7 +207,7 @@ void G1ParScanThreadState::do_oop_evac(T* p) {
markWord m = obj->mark();
if (m.is_marked()) {
- obj = cast_to_oop(m.decode_pointer());
+ obj = obj->forwardee(m);
} else {
obj = do_copy_to_survivor_space(region_attr, obj, m);
}
@@ -221,7 +221,7 @@ void G1ParScanThreadState::do_partial_array(PartialArrayScanTask task) {
oop from_obj = task.to_source_array();
assert(_g1h->is_in_reserved(from_obj), "must be in heap.");
- assert(from_obj->is_objArray(), "must be obj array");
+ assert(from_obj->forward_safe_klass()->is_objArray_klass(), "must be obj array");
assert(from_obj->is_forwarded(), "must be forwarded");
oop to_obj = from_obj->forwardee();
@@ -251,7 +251,7 @@ MAYBE_INLINE_EVACUATION
void G1ParScanThreadState::start_partial_objarray(G1HeapRegionAttr dest_attr,
oop from_obj,
oop to_obj) {
- assert(from_obj->is_objArray(), "precondition");
+ assert(from_obj->forward_safe_klass()->is_objArray_klass(), "precondition");
assert(from_obj->is_forwarded(), "precondition");
assert(from_obj->forwardee() == to_obj, "precondition");
assert(from_obj != to_obj, "should not be scanning self-forwarded objects");
@@ -378,22 +378,22 @@ G1HeapRegionAttr G1ParScanThreadState::next_region_attr(G1HeapRegionAttr const r
}
void G1ParScanThreadState::report_promotion_event(G1HeapRegionAttr const dest_attr,
- oop const old, size_t word_sz, uint age,
+ Klass* klass, size_t word_sz, uint age,
HeapWord * const obj_ptr, uint node_index) const {
PLAB* alloc_buf = _plab_allocator->alloc_buffer(dest_attr, node_index);
if (alloc_buf->contains(obj_ptr)) {
- _g1h->gc_tracer_stw()->report_promotion_in_new_plab_event(old->klass(), word_sz * HeapWordSize, age,
+ _g1h->gc_tracer_stw()->report_promotion_in_new_plab_event(klass, word_sz * HeapWordSize, age,
dest_attr.type() == G1HeapRegionAttr::Old,
alloc_buf->word_sz() * HeapWordSize);
} else {
- _g1h->gc_tracer_stw()->report_promotion_outside_plab_event(old->klass(), word_sz * HeapWordSize, age,
+ _g1h->gc_tracer_stw()->report_promotion_outside_plab_event(klass, word_sz * HeapWordSize, age,
dest_attr.type() == G1HeapRegionAttr::Old);
}
}
NOINLINE
HeapWord* G1ParScanThreadState::allocate_copy_slow(G1HeapRegionAttr* dest_attr,
- oop old,
+ Klass* klass,
size_t word_sz,
uint age,
uint node_index) {
@@ -416,7 +416,7 @@ HeapWord* G1ParScanThreadState::allocate_copy_slow(G1HeapRegionAttr* dest_attr,
update_numa_stats(node_index);
if (_g1h->gc_tracer_stw()->should_report_promotion_events()) {
// The events are checked individually as part of the actual commit
- report_promotion_event(*dest_attr, old, word_sz, age, obj_ptr, node_index);
+ report_promotion_event(*dest_attr, klass, word_sz, age, obj_ptr, node_index);
}
}
return obj_ptr;
@@ -453,7 +453,13 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
// Get the klass once. We'll need it again later, and this avoids
// re-decoding when it's compressed.
- Klass* klass = old->klass();
+ // NOTE: With compact headers, it is not safe to load the Klass* from o, because
+ // that would access the mark-word, and the mark-word might change at any time by
+ // concurrent promotion. The promoted mark-word would point to the forwardee, which
+ // may not yet have completed copying. Therefore we must load the Klass* from
+ // the mark-word that we have already loaded. This is safe, because we have checked
+ // that this is not yet forwarded in the caller.
+ Klass* klass = old->forward_safe_klass(old_mark);
const size_t word_sz = old->size_given_klass(klass);
uint age = 0;
@@ -466,7 +472,7 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
// PLAB allocations should succeed most of the time, so we'll
// normally check against null once and that's it.
if (obj_ptr == nullptr) {
- obj_ptr = allocate_copy_slow(&dest_attr, old, word_sz, age, node_index);
+ obj_ptr = allocate_copy_slow(&dest_attr, klass, word_sz, age, node_index);
if (obj_ptr == nullptr) {
// This will either forward-to-self, or detect that someone else has
// installed a forwarding pointer.
@@ -620,7 +626,7 @@ NOINLINE
oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz) {
assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));
- oop forward_ptr = old->forward_to_atomic(old, m, memory_order_relaxed);
+ oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed);
if (forward_ptr == nullptr) {
// Forward-to-self succeeded. We are the "owner" of the object.
HeapRegion* r = _g1h->heap_region_containing(old);
@@ -161,7 +161,7 @@ private:
void start_partial_objarray(G1HeapRegionAttr dest_dir, oop from, oop to);
HeapWord* allocate_copy_slow(G1HeapRegionAttr* dest_attr,
- oop old,
+ Klass* klass,
size_t word_sz,
uint age,
uint node_index);
@@ -196,7 +196,7 @@ private:
inline G1HeapRegionAttr next_region_attr(G1HeapRegionAttr const region_attr, markWord const m, uint& age);
void report_promotion_event(G1HeapRegionAttr const dest_attr,
- oop const old, size_t word_sz, uint age,
+ Klass* klass, size_t word_sz, uint age,
HeapWord * const obj_ptr, uint node_index) const;
void trim_queue_to_threshold(uint threshold);
@@ -240,15 +240,16 @@ void MutableSpace::object_iterate(ObjectClosure* cl) {
// When promotion-failure occurs during Young GC, eden/from space is not cleared,
// so we can encounter objects with "forwarded" markword.
// They are essentially dead, so skipping them
- if (!obj->is_forwarded()) {
- cl->do_object(obj);
- }
-#ifdef ASSERT
- else {
+ if (obj->is_forwarded()) {
assert(obj->forwardee() != obj, "must not be self-forwarded");
+ // It is safe to use the forwardee here. Parallel GC only uses
+ // header-based forwarding during promotion. Full GC doesn't
+ // use the object header for forwarding at all.
+ p += obj->forwardee()->size();
+ } else {
+ cl->do_object(obj);
+ p += obj->size();
}
-#endif
- p += cast_to_oop(p)->size();
}
}
@@ -400,7 +400,9 @@ class VerifyObjectStartArrayClosure : public ObjectClosure {
_start_array(start_array) { }
virtual void do_object(oop obj) {
- HeapWord* test_addr = cast_from_oop<HeapWord*>(obj) + 1;
+ // With compact headers, the objects can be one-word sized.
+ size_t int_off = AARCH64_ONLY(UseCompactObjectHeaders ? MIN2((size_t)1, obj->size() - 1) :) 1;
+ HeapWord* test_addr = cast_from_oop<HeapWord*>(obj) + int_off;
guarantee(_start_array->object_start(test_addr) == cast_from_oop<HeapWord*>(obj), "ObjectStartArray cannot find start of object");
guarantee(_start_array->is_block_allocated(cast_from_oop<HeapWord*>(obj)), "ObjectStartArray missing block allocation");
}
@@ -295,7 +295,7 @@ void PSPromotionManager::process_array_chunk(PartialArrayScanTask task) {
assert(PSChunkLargeArrays, "invariant");
oop old = task.to_source_array();
- assert(old->is_objArray(), "invariant");
+ assert(old->forward_safe_klass()->is_objArray_klass(), "invariant");
assert(old->is_forwarded(), "invariant");
TASKQUEUE_STATS_ONLY(++_array_chunks_processed);
@@ -333,7 +333,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) {
// this started. If it is the same (i.e., no forwarding
// pointer has been installed), then this thread owns
// it.
- if (obj->forward_to_atomic(obj, obj_mark) == nullptr) {
+ if (obj->forward_to_self_atomic(obj_mark) == nullptr) {
// We won any races, we "own" this object.
assert(obj == obj->forwardee(), "Sanity");
@@ -105,7 +105,7 @@ class PSPromotionManager {
void push_depth(ScannerTask task);
- inline void promotion_trace_event(oop new_obj, oop old_obj, size_t obj_size,
+ inline void promotion_trace_event(oop new_obj, Klass* klass, size_t obj_size,
uint age, bool tenured,
const PSPromotionLAB* lab);
@@ -63,7 +63,7 @@ inline void PSPromotionManager::claim_or_forward_depth(T* p) {
push_depth(ScannerTask(p));
}
-inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj,
+inline void PSPromotionManager::promotion_trace_event(oop new_obj, Klass* klass,
size_t obj_size,
uint age, bool tenured,
const PSPromotionLAB* lab) {
@@ -76,14 +76,14 @@ inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj,
if (gc_tracer->should_report_promotion_in_new_plab_event()) {
size_t obj_bytes = obj_size * HeapWordSize;
size_t lab_size = lab->capacity();
- gc_tracer->report_promotion_in_new_plab_event(old_obj->klass(), obj_bytes,
+ gc_tracer->report_promotion_in_new_plab_event(klass, obj_bytes,
age, tenured, lab_size);
}
} else {
// Promotion of object directly to heap
if (gc_tracer->should_report_promotion_outside_plab_event()) {
size_t obj_bytes = obj_size * HeapWordSize;
- gc_tracer->report_promotion_outside_plab_event(old_obj->klass(), obj_bytes,
+ gc_tracer->report_promotion_outside_plab_event(klass, obj_bytes,
age, tenured);
}
}
@@ -152,7 +152,7 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
// other thread.
OrderAccess::acquire();
// Return the already installed forwardee.
- return cast_to_oop(m.decode_pointer());
+ return o->forwardee(m);
}
}
@@ -168,7 +168,14 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
oop new_obj = nullptr;
bool new_obj_is_tenured = false;
- size_t new_obj_size = o->size();
+ // NOTE: With compact headers, it is not safe to load the Klass* from o, because
+ // that would access the mark-word, and the mark-word might change at any time by
+ // concurrent promotion. The promoted mark-word would point to the forwardee, which
+ // may not yet have completed copying. Therefore we must load the Klass* from
+ // the mark-word that we have already loaded. This is safe, because we have checked
+ // that this is not yet forwarded in the caller.
+ Klass* klass = o->forward_safe_klass(test_mark);
+ size_t new_obj_size = o->size_given_klass(klass);
// Find the objects age, MT safe.
uint age = (test_mark.has_displaced_mark_helper() /* o->has_displaced_mark() */) ?
@@ -183,7 +190,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
if (new_obj_size > (YoungPLABSize / 2)) {
// Allocate this object directly
new_obj = cast_to_oop(young_space()->cas_allocate(new_obj_size));
- promotion_trace_event(new_obj, o, new_obj_size, age, false, nullptr);
+ promotion_trace_event(new_obj, klass, new_obj_size, age, false, nullptr);
} else {
// Flush and fill
_young_lab.flush();
@@ -193,7 +200,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
_young_lab.initialize(MemRegion(lab_base, YoungPLABSize));
// Try the young lab allocation again.
new_obj = cast_to_oop(_young_lab.allocate(new_obj_size));
- promotion_trace_event(new_obj, o, new_obj_size, age, false, &_young_lab);
+ promotion_trace_event(new_obj, klass, new_obj_size, age, false, &_young_lab);
} else {
_young_gen_is_full = true;
}
@@ -219,7 +226,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
if (new_obj_size > (OldPLABSize / 2)) {
// Allocate this object directly
new_obj = cast_to_oop(old_gen()->allocate(new_obj_size));
- promotion_trace_event(new_obj, o, new_obj_size, age, true, nullptr);
+ promotion_trace_event(new_obj, klass, new_obj_size, age, true, nullptr);
} else {
// Flush and fill
_old_lab.flush();
@@ -229,7 +236,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
_old_lab.initialize(MemRegion(lab_base, OldPLABSize));
// Try the old lab allocation again.
new_obj = cast_to_oop(_old_lab.allocate(new_obj_size));
- promotion_trace_event(new_obj, o, new_obj_size, age, true, &_old_lab);
+ promotion_trace_event(new_obj, klass, new_obj_size, age, true, &_old_lab);
}
}
}
@@ -254,7 +261,24 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
// Parallel GC claims with a release - so other threads might access this object
// after claiming and they should see the "completed" object.
- ContinuationGCSupport::transform_stack_chunk(new_obj);
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ // The copy above is not atomic. Make sure we have seen the proper mark
+ // and re-install it into the copy, so that Klass* is guaranteed to be correct.
+ markWord mark = o->mark();
+ if (!mark.is_marked()) {
+ new_obj->set_mark(mark);
+ ContinuationGCSupport::transform_stack_chunk(new_obj);
+ } else {
+ // If we copied a mark-word that indicates 'forwarded' state, the object
+ // installation would not succeed. We cannot access Klass* anymore either.
+ // Skip the transformation.
+ }
+ } else
+#endif
+ {
+ ContinuationGCSupport::transform_stack_chunk(new_obj);
+ }
// Now we have to CAS in the header.
// Make copy visible to threads reading the forwardee.
@@ -882,7 +882,7 @@ void DefNewGeneration::remove_forwarding_pointers() {
struct ResetForwardedMarkWord : ObjectClosure {
void do_object(oop obj) override {
if (obj->is_forwarded()) {
- obj->init_mark();
+ obj->forward_safe_init_mark();
}
}
} cl;
@@ -905,8 +905,7 @@ void DefNewGeneration::handle_promotion_failure(oop old) {
ContinuationGCSupport::transform_stack_chunk(old);
- // forward to self
- old->forward_to(old);
+ old->forward_to_self();
_promo_failure_scan_stack.push(old);
@@ -46,6 +46,7 @@
#include "gc/shared/modRefBarrierSet.hpp"
#include "gc/shared/referencePolicy.hpp"
#include "gc/shared/referenceProcessorPhaseTimes.hpp"
+#include "gc/shared/slidingForwarding.hpp"
#include "gc/shared/space.hpp"
#include "gc/shared/strongRootsScope.hpp"
#include "gc/shared/weakProcessor.hpp"
@@ -88,6 +89,8 @@ void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) {
mark_sweep_phase1(clear_all_softrefs);
+ SlidingForwarding::begin();
+
mark_sweep_phase2();
// Don't add any more derived pointers during phase3
@@ -106,6 +109,8 @@ void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) {
// (Should this be in general part?)
gch->save_marks();
+ SlidingForwarding::end();
+
deallocate_stacks();
MarkSweep::_string_dedup_requests->flush();
@@ -260,15 +265,27 @@ void GenMarkSweep::mark_sweep_phase3() {
ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
- CodeBlobToOopClosure code_closure(&adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations);
- gch->process_roots(GenCollectedHeap::SO_AllCodeCache,
- &adjust_pointer_closure,
- &adjust_cld_closure,
- &adjust_cld_closure,
- &code_closure);
-
- gch->gen_process_weak_roots(&adjust_pointer_closure);
-
+ if (UseAltGCForwarding) {
+ AdjustPointerClosure<true> adjust_pointer_closure;
+ CLDToOopClosure adjust_cld_closure(&adjust_pointer_closure, ClassLoaderData::_claim_stw_fullgc_adjust);
+ CodeBlobToOopClosure code_closure(&adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations);
+ gch->process_roots(GenCollectedHeap::SO_AllCodeCache,
+ &adjust_pointer_closure,
+ &adjust_cld_closure,
+ &adjust_cld_closure,
+ &code_closure);
+ gch->gen_process_weak_roots(&adjust_pointer_closure);
+ } else {
+ AdjustPointerClosure<false> adjust_pointer_closure;
+ CLDToOopClosure adjust_cld_closure(&adjust_pointer_closure, ClassLoaderData::_claim_stw_fullgc_adjust);
+ CodeBlobToOopClosure code_closure(&adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations);
+ gch->process_roots(GenCollectedHeap::SO_AllCodeCache,
+ &adjust_pointer_closure,
+ &adjust_cld_closure,
+ &adjust_cld_closure,
+ &code_closure);
+ gch->gen_process_weak_roots(&adjust_pointer_closure);
+ }
adjust_marks();
GenAdjustPointersClosure blk;
gch->generation_iterate(&blk, true);
@@ -60,7 +60,6 @@ MarkSweep::FollowRootClosure MarkSweep::follow_root_closure;
MarkAndPushClosure MarkSweep::mark_and_push_closure(ClassLoaderData::_claim_stw_fullgc_mark);
CLDToOopClosure MarkSweep::follow_cld_closure(&mark_and_push_closure, ClassLoaderData::_claim_stw_fullgc_mark);
-CLDToOopClosure MarkSweep::adjust_cld_closure(&adjust_pointer_closure, ClassLoaderData::_claim_stw_fullgc_adjust);
template <class T> void MarkSweep::KeepAliveClosure::do_oop_work(T* p) {
mark_and_push(p);
@@ -142,8 +141,9 @@ template <class T> void MarkSweep::follow_root(T* p) {
void MarkSweep::FollowRootClosure::do_oop(oop* p) { follow_root(p); }
void MarkSweep::FollowRootClosure::do_oop(narrowOop* p) { follow_root(p); }
+template <bool ALT_FWD>
void PreservedMark::adjust_pointer() {
- MarkSweep::adjust_pointer(&_obj);
+ MarkSweep::adjust_pointer<ALT_FWD>(&_obj);
}
void PreservedMark::restore() {
@@ -172,12 +172,14 @@ void MarkSweep::mark_object(oop obj) {
_string_dedup_requests->add(obj);
}
+ // Do the transform while we still have the header intact,
+ // which might include important class information.
+ ContinuationGCSupport::transform_stack_chunk(obj);
+
// some marks may contain information we need to preserve so we store them away
// and overwrite the mark. We'll restore it at the end of markSweep.
markWord mark = obj->mark();
- obj->set_mark(markWord::prototype().set_marked());
-
- ContinuationGCSupport::transform_stack_chunk(obj);
+ obj->set_mark(obj->prototype_mark().set_marked());
if (obj->mark_must_be_preserved(mark)) {
preserve_mark(obj, mark);
@@ -200,19 +202,26 @@ void MarkAndPushClosure::do_oop_work(T* p) { MarkSweep::mark_and_push
void MarkAndPushClosure::do_oop( oop* p) { do_oop_work(p); }
void MarkAndPushClosure::do_oop(narrowOop* p) { do_oop_work(p); }
-AdjustPointerClosure MarkSweep::adjust_pointer_closure;
-
-void MarkSweep::adjust_marks() {
+template <bool ALT_FWD>
+void MarkSweep::adjust_marks_impl() {
// adjust the oops we saved earlier
for (size_t i = 0; i < _preserved_count; i++) {
- _preserved_marks[i].adjust_pointer();
+ _preserved_marks[i].adjust_pointer<ALT_FWD>();
}
// deal with the overflow stack
StackIterator<PreservedMark, mtGC> iter(_preserved_overflow_stack);
while (!iter.is_empty()) {
PreservedMark* p = iter.next_addr();
- p->adjust_pointer();
+ p->adjust_pointer<ALT_FWD>();
+ }
+}
+
+void MarkSweep::adjust_marks() {
+ if (UseAltGCForwarding) {
+ adjust_marks_impl<true>();
+ } else {
+ adjust_marks_impl<false>();
}
}
@@ -50,7 +50,6 @@ class STWGCTimer;
// declared at end
class PreservedMark;
class MarkAndPushClosure;
-class AdjustPointerClosure;
class MarkSweep : AllStatic {
//
@@ -84,7 +83,6 @@ class MarkSweep : AllStatic {
//
// Friend decls
//
- friend class AdjustPointerClosure;
friend class KeepAliveClosure;
//
@@ -124,8 +122,6 @@ class MarkSweep : AllStatic {
static MarkAndPushClosure mark_and_push_closure;
static FollowStackClosure follow_stack_closure;
static CLDToOopClosure follow_cld_closure;
- static AdjustPointerClosure adjust_pointer_closure;
- static CLDToOopClosure adjust_cld_closure;
// Accessors
static uint total_invocations() { return _total_invocations; }
@@ -141,16 +137,21 @@ class MarkSweep : AllStatic {
static void adjust_marks(); // Adjust the pointers in the preserved marks table
static void restore_marks(); // Restore the marks that we saved in preserve_mark
+ template <bool ALT_FWD>
static size_t adjust_pointers(oop obj);
static void follow_stack(); // Empty marking stack.
- template <class T> static inline void adjust_pointer(T* p);
+ template <bool ALT_FWD, class T>
+ static void adjust_pointer(T* p);
// Check mark and maybe push on marking stack
template <class T> static void mark_and_push(T* p);
private:
+ template <bool ALT_FWD>
+ static void adjust_marks_impl();
+
// Call backs for marking
static void mark_object(oop obj);
// Mark pointer and follow contents. Empty marking stack afterwards.
@@ -178,6 +179,7 @@ public:
}
};
+template <bool ALT_FWD>
class AdjustPointerClosure: public BasicOopIterateClosure {
public:
template <typename T> void do_oop_work(T* p);
@@ -193,6 +195,7 @@ private:
public:
PreservedMark(oop obj, markWord mark) : _obj(obj), _mark(mark) {}
+ template <bool ALT_FWD>
void adjust_pointer();
void restore();
};
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -31,6 +31,7 @@
#include "classfile/javaClasses.inline.hpp"
#include "gc/shared/continuationGCSupport.inline.hpp"
#include "gc/serial/serialStringDedup.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
#include "memory/universe.hpp"
#include "oops/markWord.hpp"
#include "oops/access.inline.hpp"
@@ -39,27 +40,34 @@
#include "utilities/align.hpp"
#include "utilities/stack.inline.hpp"
-template <class T> inline void MarkSweep::adjust_pointer(T* p) {
+template <bool ALT_FWD, class T>
+inline void MarkSweep::adjust_pointer(T* p) {
T heap_oop = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(heap_oop)) {
oop obj = CompressedOops::decode_not_null(heap_oop);
assert(Universe::heap()->is_in(obj), "should be in heap");
- if (obj->is_forwarded()) {
- oop new_obj = obj->forwardee();
+ if (SlidingForwarding::is_forwarded(obj)) {
+ oop new_obj = SlidingForwarding::forwardee<ALT_FWD>(obj);
assert(is_object_aligned(new_obj), "oop must be aligned");
RawAccess<IS_NOT_NULL>::oop_store(p, new_obj);
}
}
}
+template <bool ALT_FWD>
template <typename T>
-void AdjustPointerClosure::do_oop_work(T* p) { MarkSweep::adjust_pointer(p); }
-inline void AdjustPointerClosure::do_oop(oop* p) { do_oop_work(p); }
-inline void AdjustPointerClosure::do_oop(narrowOop* p) { do_oop_work(p); }
+void AdjustPointerClosure<ALT_FWD>::do_oop_work(T* p) { MarkSweep::adjust_pointer<ALT_FWD>(p); }
+template <bool ALT_FWD>
+inline void AdjustPointerClosure<ALT_FWD>::do_oop(oop* p) { do_oop_work(p); }
+template <bool ALT_FWD>
+inline void AdjustPointerClosure<ALT_FWD>::do_oop(narrowOop* p) { do_oop_work(p); }
+
+template <bool ALT_FWD>
inline size_t MarkSweep::adjust_pointers(oop obj) {
- return obj->oop_iterate_size(&MarkSweep::adjust_pointer_closure);
+ AdjustPointerClosure<ALT_FWD> adjust_pointer_closure;
+ return obj->oop_iterate_size(&adjust_pointer_closure);
}
#endif // SHARE_GC_SERIAL_MARKSWEEP_INLINE_HPP
@@ -228,7 +228,9 @@ bool CollectedHeap::is_oop(oop object) const {
return false;
}
- if (!Metaspace::contains(object->klass_raw())) {
+ // With compact headers, we can't safely access the class, due
+ // to possibly forwarded objects.
+ if (AARCH64_ONLY(!UseCompactObjectHeaders &&) !Metaspace::contains(object->klass_raw())) {
return false;
}
@@ -401,6 +403,13 @@ void CollectedHeap::set_gc_cause(GCCause::Cause v) {
_gc_cause = v;
}
+// Returns the header size in words aligned to the requirements of the
+// array object type.
+static int int_array_header_size() {
+ size_t typesize_in_bytes = arrayOopDesc::header_size_in_bytes();
+ return (int)align_up(typesize_in_bytes, HeapWordSize)/HeapWordSize;
+}
+
size_t CollectedHeap::max_tlab_size() const {
// TLABs can't be bigger than we can fill with a int[Integer.MAX_VALUE].
// This restriction could be removed by enabling filling with multiple arrays.
@@ -410,14 +419,14 @@ size_t CollectedHeap::max_tlab_size() const {
// We actually lose a little by dividing first,
// but that just makes the TLAB somewhat smaller than the biggest array,
// which is fine, since we'll be able to fill that.
- size_t max_int_size = typeArrayOopDesc::header_size(T_INT) +
+ size_t max_int_size = int_array_header_size() +
sizeof(jint) *
((juint) max_jint / (size_t) HeapWordSize);
return align_down(max_int_size, MinObjAlignment);
}
size_t CollectedHeap::filler_array_hdr_size() {
- return align_object_offset(arrayOopDesc::header_size(T_INT)); // align to Long
+ return align_object_offset(int_array_header_size()); // align to Long
}
size_t CollectedHeap::filler_array_min_size() {
@@ -309,7 +309,7 @@ class CollectedHeap : public CHeapObj<mtGC> {
}
virtual void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap);
- static constexpr size_t min_dummy_object_size() {
+ static size_t min_dummy_object_size() {
return oopDesc::header_size();
}
@@ -690,8 +690,12 @@
product(uint, GCCardSizeInBytes, 512, \
"Card table entry size (in bytes) for card based collectors") \
range(128, NOT_LP64(512) LP64_ONLY(1024)) \
- constraint(GCCardSizeInBytesConstraintFunc,AtParse)
- // end of GC_FLAGS
+ constraint(GCCardSizeInBytesConstraintFunc,AtParse) \
+ \
+ product(bool, UseAltGCForwarding, false, EXPERIMENTAL, \
+ "Use alternative GC forwarding that preserves object headers") \
+
+// end of GC_FLAGS
DECLARE_FLAGS(GC_FLAGS)
@@ -55,6 +55,7 @@
#include "gc/shared/oopStorageParState.inline.hpp"
#include "gc/shared/oopStorageSet.inline.hpp"
#include "gc/shared/scavengableNMethods.hpp"
+#include "gc/shared/slidingForwarding.hpp"
#include "gc/shared/space.hpp"
#include "gc/shared/strongRootsScope.hpp"
#include "gc/shared/weakProcessor.hpp"
@@ -133,6 +134,8 @@ jint GenCollectedHeap::initialize() {
GCInitLogger::print();
+ SlidingForwarding::initialize(_reserved, SpaceAlignment / HeapWordSize);
+
return JNI_OK;
}
@@ -377,18 +377,26 @@ void MemAllocator::mem_clear(HeapWord* mem) const {
assert(mem != nullptr, "cannot initialize null object");
const size_t hs = oopDesc::header_size();
assert(_word_size >= hs, "unexpected object size");
- oopDesc::set_klass_gap(mem, 0);
+ if (AARCH64_ONLY(!UseCompactObjectHeaders) NOT_AARCH64(true)) {
+ oopDesc::set_klass_gap(mem, 0);
+ }
Copy::fill_to_aligned_words(mem + hs, _word_size - hs);
}
oop MemAllocator::finish(HeapWord* mem) const {
assert(mem != nullptr, "null object pointer");
- // May be bootstrapping
- oopDesc::set_mark(mem, markWord::prototype());
// Need a release store to ensure array/class length, mark word, and
// object zeroing are visible before setting the klass non-null, for
// concurrent collectors.
- oopDesc::release_set_klass(mem, _klass);
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ oopDesc::release_set_mark(mem, _klass->prototype_header());
+ } else
+#endif
+ {
+ oopDesc::set_mark(mem, markWord::prototype());
+ oopDesc::release_set_klass(mem, _klass);
+ }
return cast_to_oop(mem);
}
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
#include "precompiled.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
#include "gc/shared/workerThread.hpp"
#include "gc/shared/workerUtils.hpp"
#include "memory/allocation.inline.hpp"
@@ -40,18 +41,27 @@ void PreservedMarks::restore() {
assert_empty();
}
-void PreservedMarks::adjust_during_full_gc() {
+template <bool ALT_FWD>
+void PreservedMarks::adjust_during_full_gc_impl() {
StackIterator<OopAndMarkWord, mtGC> iter(_stack);
while (!iter.is_empty()) {
OopAndMarkWord* elem = iter.next_addr();
oop obj = elem->get_oop();
if (obj->is_forwarded()) {
- elem->set_oop(obj->forwardee());
+ elem->set_oop(SlidingForwarding::forwardee<ALT_FWD>(obj));
}
}
}
+void PreservedMarks::adjust_during_full_gc() {
+ if (UseAltGCForwarding) {
+ adjust_during_full_gc_impl<true>();
+ } else {
+ adjust_during_full_gc_impl<false>();
+ }
+}
+
void PreservedMarks::restore_and_increment(volatile size_t* const total_size_addr) {
const size_t stack_size = size();
restore();
@@ -54,6 +54,9 @@ private:
inline bool should_preserve_mark(oop obj, markWord m) const;
+ template <bool ALT_FWD>
+ void adjust_during_full_gc_impl();
+
public:
size_t size() const { return _stack.size(); }
inline void push_if_necessary(oop obj, markWord m);
@@ -26,6 +26,7 @@
#define SHARE_GC_SHARED_PRESERVEDMARKS_INLINE_HPP
#include "gc/shared/preservedMarks.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
#include "logging/log.hpp"
#include "oops/oop.inline.hpp"
new file mode 100644
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/shared/slidingForwarding.hpp"
+#include "utilities/ostream.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+// We cannot use 0, because that may already be a valid base address in zero-based heaps.
+// 0x1 is safe because heap base addresses must be aligned by much larger alignment
+HeapWord* const SlidingForwarding::UNUSED_BASE = reinterpret_cast<HeapWord*>(0x1);
+
+HeapWord* SlidingForwarding::_heap_start = nullptr;
+size_t SlidingForwarding::_region_size_words = 0;
+size_t SlidingForwarding::_heap_start_region_bias = 0;
+size_t SlidingForwarding::_num_regions = 0;
+uint SlidingForwarding::_region_size_bytes_shift = 0;
+uintptr_t SlidingForwarding::_region_mask = 0;
+HeapWord** SlidingForwarding::_biased_bases[SlidingForwarding::NUM_TARGET_REGIONS] = { nullptr, nullptr };
+HeapWord** SlidingForwarding::_bases_table = nullptr;
+SlidingForwarding::FallbackTable* SlidingForwarding::_fallback_table = nullptr;
+
+void SlidingForwarding::initialize(MemRegion heap, size_t region_size_words) {
+#ifdef _LP64
+ if (UseAltGCForwarding) {
+ _heap_start = heap.start();
+
+ // If the heap is small enough to fit directly into the available offset bits,
+ // and we are running Serial GC, we can treat the whole heap as a single region
+ // if it happens to be aligned to allow biasing.
+ size_t rounded_heap_size = round_up_power_of_2(heap.byte_size());
+
+ if (UseSerialGC && (heap.word_size() <= (1 << NUM_OFFSET_BITS)) &&
+ is_aligned((uintptr_t)_heap_start, rounded_heap_size)) {
+ _num_regions = 1;
+ _region_size_words = heap.word_size();
+ _region_size_bytes_shift = log2i_exact(rounded_heap_size);
+ } else {
+ _num_regions = align_up(pointer_delta(heap.end(), heap.start()), region_size_words) / region_size_words;
+ _region_size_words = region_size_words;
+ _region_size_bytes_shift = log2i_exact(_region_size_words) + LogHeapWordSize;
+ }
+ _heap_start_region_bias = (uintptr_t)_heap_start >> _region_size_bytes_shift;
+ _region_mask = ~((uintptr_t(1) << _region_size_bytes_shift) - 1);
+
+ guarantee((_heap_start_region_bias << _region_size_bytes_shift) == (uintptr_t)_heap_start, "must be aligned: _heap_start_region_bias: " SIZE_FORMAT ", _region_size_byte_shift: %u, _heap_start: " PTR_FORMAT, _heap_start_region_bias, _region_size_bytes_shift, p2i(_heap_start));
+
+ assert(_region_size_words >= 1, "regions must be at least a word large");
+ assert(_bases_table == nullptr, "should not be initialized yet");
+ assert(_fallback_table == nullptr, "should not be initialized yet");
+ }
+#endif
+}
+
+void SlidingForwarding::begin() {
+#ifdef _LP64
+ if (UseAltGCForwarding) {
+ assert(_bases_table == nullptr, "should not be initialized yet");
+ assert(_fallback_table == nullptr, "should not be initialized yet");
+
+ size_t max = _num_regions * NUM_TARGET_REGIONS;
+ _bases_table = NEW_C_HEAP_ARRAY(HeapWord*, max, mtGC);
+ HeapWord** biased_start = _bases_table - _heap_start_region_bias;
+ _biased_bases[0] = biased_start;
+ _biased_bases[1] = biased_start + _num_regions;
+ for (size_t i = 0; i < max; i++) {
+ _bases_table[i] = UNUSED_BASE;
+ }
+ }
+#endif
+}
+
+void SlidingForwarding::end() {
+#ifdef _LP64
+ if (UseAltGCForwarding) {
+ assert(_bases_table != nullptr, "should be initialized");
+ FREE_C_HEAP_ARRAY(HeapWord*, _bases_table);
+ _bases_table = nullptr;
+ delete _fallback_table;
+ _fallback_table = nullptr;
+ }
+#endif
+}
+
+void SlidingForwarding::fallback_forward_to(HeapWord* from, HeapWord* to) {
+ if (_fallback_table == nullptr) {
+ _fallback_table = new (mtGC) FallbackTable();
+ }
+ _fallback_table->put_when_absent(from, to);
+}
+
+HeapWord* SlidingForwarding::fallback_forwardee(HeapWord* from) {
+ assert(_fallback_table != nullptr, "fallback table must be present");
+ HeapWord** found = _fallback_table->get(from);
+ if (found != nullptr) {
+ return *found;
+ } else {
+ return nullptr;
+ }
+}
new file mode 100644
@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHARED_SLIDINGFORWARDING_HPP
+#define SHARE_GC_SHARED_SLIDINGFORWARDING_HPP
+
+#include "memory/allocation.hpp"
+#include "memory/memRegion.hpp"
+#include "oops/markWord.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "utilities/fastHash.hpp"
+#include "utilities/resourceHash.hpp"
+
+/**
+ * SlidingForwarding is a method to store forwarding information in a compressed form into the object header,
+ * that has been specifically designed for sliding compaction GCs and compact object headers. With compact object
+ * headers, we store the compressed class pointer in the header, which would be overwritten by full forwarding
+ * pointer, if we allow the legacy forwarding code to act. This would lose the class information for the object,
+ * which is required later in GC cycle to iterate the reference fields and get the object size for copying.
+ *
+ * SlidingForwarding requires only small side tables and guarantees constant-time access and modification.
+ *
+ * The idea is to use a pointer compression scheme very similar to the one that is used for compressed oops.
+ * We divide the heap into number of logical regions. Each region spans maximum of 2^NUM_OFFSET_BITS words.
+ *
+ * The key advantage of sliding compaction for encoding efficiency: it can forward objects from one region to a
+ * maximum of two regions. This is an intuitive property: when we slide the compact region full of data, it can
+ * only span two adjacent regions. This property allows us to use the off-side table to record the addresses of
+ * two target regions. The table holds N*2 entries for N logical regions. For each region, it gives the base
+ * address of the two target regions, or a special placeholder if not used. A single bit in forwarding would
+ * indicate to which of the two "to" regions the object is forwarded into.
+ *
+ * This encoding efficiency allows to store the forwarding information in the object header _together_ with the
+ * compressed class pointer.
+ *
+ * When recording the sliding forwarding, the mark word would look roughly like this:
+ *
+ * 64 32 0
+ * [................................OOOOOOOOOOOOOOOOOOOOOOOOOOOOAFTT]
+ * ^----- normal lock bits, would record "object is forwarded"
+ * ^------- fallback bit (explained below)
+ * ^-------- alternate region select
+ * ^------------------------------------ in-region offset
+ * ^-------------------------------------------------------------------- protected area, *not touched* by this code, useful for
+ * compressed class pointer with compact object headers
+ *
+ * Adding a forwarding then generally works as follows:
+ * 1. Compute the "to" offset in the "to" region, this gives "offset".
+ * 2. Check if the primary "from" offset at base table contains "to" region base, use it.
+ * If not usable, continue to next step. If usable, set "alternate" = "false" and jump to (4).
+ * 3. Check if the alternate "from" offset at base table contains "to" region base, use it.
+ * This gives us "alternate" = "true". This should always complete for sliding forwarding.
+ * 4. Compute the mark word from "offset" and "alternate", write it out
+ *
+ * Similarly, looking up the target address, given an original object address generally works as follows:
+ * 1. Load the mark from object, and decode "offset" and "alternate" from there
+ * 2. Compute the "from" base offset from the object
+ * 3. Look up "to" region base from the base table either at primary or alternate indices, using "alternate" flag
+ * 4. Compute the "to" address from "to" region base and "offset"
+ *
+ * This algorithm is broken by G1 last-ditch serial compaction: there, object from a single region can be
+ * forwarded to multiple, more than two regions. To deal with that, we initialize a fallback-hashtable for
+ * storing those extra forwardings, and set another bit in the header to indicate that the forwardee is not
+ * encoded but should be looked-up in the hashtable. G1 serial compaction is not very common - it is the
+ * last-last-ditch GC that is used when the JVM is scrambling to squeeze more space out of the heap, and at
+ * that point, ultimate performance is no longer the main concern.
+ */
+class SlidingForwarding : public AllStatic {
+private:
+
+ /*
+ * A simple hash-table that acts as fallback for the sliding forwarding.
+ * This is used in the case of G1 serial compaction, which violates the
+ * assumption of sliding forwarding that each object of any region is only
+ * ever forwarded to one of two target regions. At this point, the GC is
+ * scrambling to free up more Java heap memory, and therefore performance
+ * is not the major concern.
+ *
+ * The implementation is a straightforward open hashtable.
+ * It is a single-threaded (not thread-safe) implementation, and that
+ * is sufficient because G1 serial compaction is single-threaded.
+ */
+ inline static unsigned hash(HeapWord* const& from) {
+ uint64_t val = reinterpret_cast<uint64_t>(from);
+ uint64_t hash = FastHash::get_hash64(val, UCONST64(0xAAAAAAAAAAAAAAAA));
+ return checked_cast<unsigned>(hash >> 32);
+ }
+ inline static bool equals(HeapWord* const& lhs, HeapWord* const& rhs) {
+ return lhs == rhs;
+ }
+ typedef ResourceHashtable<HeapWord* /* key-type */, HeapWord* /* value-type */,
+ 1024 /* size */, AnyObj::C_HEAP /* alloc-type */, mtGC,
+ SlidingForwarding::hash, SlidingForwarding::equals> FallbackTable;
+
+ static const uintptr_t MARK_LOWER_HALF_MASK = right_n_bits(32);
+
+ // We need the lowest two bits to indicate a forwarded object.
+ // The next bit indicates that the forwardee should be looked-up in a fallback-table.
+ static const int FALLBACK_SHIFT = markWord::lock_bits;
+ static const int FALLBACK_BITS = 1;
+ static const int FALLBACK_MASK = right_n_bits(FALLBACK_BITS) << FALLBACK_SHIFT;
+
+ // Next bit selects the target region
+ static const int ALT_REGION_SHIFT = FALLBACK_SHIFT + FALLBACK_BITS;
+ static const int ALT_REGION_BITS = 1;
+ // This will be "2" always, but expose it as named constant for clarity
+ static const size_t NUM_TARGET_REGIONS = 1 << ALT_REGION_BITS;
+
+ // The offset bits start then
+ static const int OFFSET_BITS_SHIFT = ALT_REGION_SHIFT + ALT_REGION_BITS;
+
+ // How many bits we use for the offset
+ static const int NUM_OFFSET_BITS = 32 - OFFSET_BITS_SHIFT;
+
+ // Indicates an unused base address in the target base table.
+ static HeapWord* const UNUSED_BASE;
+
+ static HeapWord* _heap_start;
+ static size_t _region_size_words;
+
+ static size_t _heap_start_region_bias;
+ static size_t _num_regions;
+ static uint _region_size_bytes_shift;
+ static uintptr_t _region_mask;
+
+ // The target base table memory.
+ static HeapWord** _bases_table;
+ // Entries into the target base tables, biased to the start of the heap.
+ static HeapWord** _biased_bases[NUM_TARGET_REGIONS];
+
+ static FallbackTable* _fallback_table;
+
+ static inline size_t biased_region_index_containing(HeapWord* addr);
+
+ static inline uintptr_t encode_forwarding(HeapWord* from, HeapWord* to);
+ static inline HeapWord* decode_forwarding(HeapWord* from, uintptr_t encoded);
+
+ static void fallback_forward_to(HeapWord* from, HeapWord* to);
+ static HeapWord* fallback_forwardee(HeapWord* from);
+
+ static inline void forward_to_impl(oop from, oop to);
+ static inline oop forwardee_impl(oop from);
+
+public:
+ static void initialize(MemRegion heap, size_t region_size_words);
+
+ static void begin();
+ static void end();
+
+ static inline bool is_forwarded(oop obj);
+ static inline bool is_not_forwarded(oop obj);
+
+ template <bool ALT_FWD>
+ static inline void forward_to(oop from, oop to);
+ template <bool ALT_FWD>
+ static inline oop forwardee(oop from);
+};
+
+#endif // SHARE_GC_SHARED_SLIDINGFORWARDING_HPP
new file mode 100644
@@ -0,0 +1,171 @@
+/*
+ * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_SHARED_SLIDINGFORWARDING_INLINE_HPP
+#define SHARE_GC_SHARED_SLIDINGFORWARDING_INLINE_HPP
+
+#include "gc/shared/gc_globals.hpp"
+#include "gc/shared/slidingForwarding.hpp"
+#include "oops/markWord.hpp"
+#include "oops/oop.inline.hpp"
+#include "utilities/macros.hpp"
+
+inline bool SlidingForwarding::is_forwarded(oop obj) {
+ return obj->is_forwarded();
+}
+
+inline bool SlidingForwarding::is_not_forwarded(oop obj) {
+ return !obj->is_forwarded();
+}
+
+size_t SlidingForwarding::biased_region_index_containing(HeapWord* addr) {
+ return (uintptr_t)addr >> _region_size_bytes_shift;
+}
+
+uintptr_t SlidingForwarding::encode_forwarding(HeapWord* from, HeapWord* to) {
+ static_assert(NUM_TARGET_REGIONS == 2, "Only implemented for this amount");
+
+ size_t from_reg_idx = biased_region_index_containing(from);
+ HeapWord* to_region_base = (HeapWord*)((uintptr_t)to & _region_mask);
+
+ HeapWord** base = &_biased_bases[0][from_reg_idx];
+ uintptr_t alternate = 0;
+ if (*base == to_region_base) {
+ // Primary is good
+ } else if (*base == UNUSED_BASE) {
+ // Primary is free
+ *base = to_region_base;
+ } else {
+ base = &_biased_bases[1][from_reg_idx];
+ if (*base == to_region_base) {
+ // Alternate is good
+ } else if (*base == UNUSED_BASE) {
+ // Alternate is free
+ *base = to_region_base;
+ } else {
+ // Both primary and alternate are not fitting
+ // This happens only in the following rare situations:
+ // - In Serial GC, sometimes when compact-top switches spaces, because the
+ // region boudaries are virtual and objects can cross regions
+ // - In G1 serial compaction, because tails of various compaction chains
+ // are distributed across the remainders of already compacted regions.
+ return (1 << FALLBACK_SHIFT) | markWord::marked_value;
+ }
+ alternate = 1;
+ }
+
+ size_t offset = pointer_delta(to, to_region_base);
+ assert(offset < _region_size_words, "Offset should be within the region. from: " PTR_FORMAT
+ ", to: " PTR_FORMAT ", to_region_base: " PTR_FORMAT ", offset: " SIZE_FORMAT,
+ p2i(from), p2i(to), p2i(to_region_base), offset);
+
+ uintptr_t encoded = (offset << OFFSET_BITS_SHIFT) |
+ (alternate << ALT_REGION_SHIFT) |
+ markWord::marked_value;
+
+ assert(to == decode_forwarding(from, encoded), "must be reversible");
+ assert((encoded & ~MARK_LOWER_HALF_MASK) == 0, "must encode to lowest 32 bits");
+ return encoded;
+}
+
+HeapWord* SlidingForwarding::decode_forwarding(HeapWord* from, uintptr_t encoded) {
+ assert((encoded & markWord::lock_mask_in_place) == markWord::marked_value, "must be marked as forwarded");
+ assert((encoded & FALLBACK_MASK) == 0, "must not be fallback-forwarded");
+ assert((encoded & ~MARK_LOWER_HALF_MASK) == 0, "must decode from lowest 32 bits");
+ size_t alternate = (encoded >> ALT_REGION_SHIFT) & right_n_bits(ALT_REGION_BITS);
+ assert(alternate < NUM_TARGET_REGIONS, "Sanity");
+ uintptr_t offset = (encoded >> OFFSET_BITS_SHIFT);
+
+ size_t from_idx = biased_region_index_containing(from);
+ HeapWord* base = _biased_bases[alternate][from_idx];
+ assert(base != UNUSED_BASE, "must not be unused base");
+ HeapWord* decoded = base + offset;
+ assert(decoded >= _heap_start,
+ "Address must be above heap start. encoded: " INTPTR_FORMAT ", alt_region: " SIZE_FORMAT ", base: " PTR_FORMAT,
+ encoded, alternate, p2i(base));
+
+ return decoded;
+}
+
+inline void SlidingForwarding::forward_to_impl(oop from, oop to) {
+ assert(_bases_table != nullptr, "call begin() before forwarding");
+
+ markWord from_header = from->mark();
+ if (from_header.has_displaced_mark_helper()) {
+ from_header = from_header.displaced_mark_helper();
+ }
+
+ HeapWord* from_hw = cast_from_oop<HeapWord*>(from);
+ HeapWord* to_hw = cast_from_oop<HeapWord*>(to);
+ uintptr_t encoded = encode_forwarding(from_hw, to_hw);
+ markWord new_header = markWord((from_header.value() & ~MARK_LOWER_HALF_MASK) | encoded);
+ from->set_mark(new_header);
+
+ if ((encoded & FALLBACK_MASK) != 0) {
+ fallback_forward_to(from_hw, to_hw);
+ }
+}
+
+template <bool ALT_FWD>
+inline void SlidingForwarding::forward_to(oop obj, oop fwd) {
+#ifdef _LP64
+ if (ALT_FWD) {
+ assert(_bases_table != nullptr, "expect sliding forwarding initialized");
+ forward_to_impl(obj, fwd);
+ assert(forwardee<ALT_FWD>(obj) == fwd, "must be forwarded to correct forwardee");
+ } else
+#endif
+ {
+ obj->forward_to(fwd);
+ }
+}
+
+inline oop SlidingForwarding::forwardee_impl(oop from) {
+ assert(_bases_table != nullptr, "call begin() before asking for forwarding");
+
+ markWord header = from->mark();
+ HeapWord* from_hw = cast_from_oop<HeapWord*>(from);
+ if ((header.value() & FALLBACK_MASK) != 0) {
+ HeapWord* to = fallback_forwardee(from_hw);
+ return cast_to_oop(to);
+ }
+ uintptr_t encoded = header.value() & MARK_LOWER_HALF_MASK;
+ HeapWord* to = decode_forwarding(from_hw, encoded);
+ return cast_to_oop(to);
+}
+
+template <bool ALT_FWD>
+inline oop SlidingForwarding::forwardee(oop obj) {
+#ifdef _LP64
+ if (ALT_FWD) {
+ assert(_bases_table != nullptr, "expect sliding forwarding initialized");
+ return forwardee_impl(obj);
+ } else
+#endif
+ {
+ return obj->forwardee();
+ }
+}
+
+#endif // SHARE_GC_SHARED_SLIDINGFORWARDING_INLINE_HPP
@@ -27,6 +27,7 @@
#include "classfile/vmSymbols.hpp"
#include "gc/shared/collectedHeap.inline.hpp"
#include "gc/shared/genCollectedHeap.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
#include "gc/shared/space.hpp"
#include "gc/shared/space.inline.hpp"
#include "gc/shared/spaceDecorator.inline.hpp"
@@ -244,7 +245,7 @@ void ContiguousSpace::mangle_unused_area_complete() {
}
#endif // NOT_PRODUCT
-
+template <bool ALT_FWD>
HeapWord* ContiguousSpace::forward(oop q, size_t size,
CompactPoint* cp, HeapWord* compact_top) {
// q is alive
@@ -269,13 +270,13 @@ HeapWord* ContiguousSpace::forward(oop q, size_t size,
// store the forwarding pointer into the mark word
if (cast_from_oop<HeapWord*>(q) != compact_top) {
- q->forward_to(cast_to_oop(compact_top));
+ SlidingForwarding::forward_to<ALT_FWD>(q, cast_to_oop(compact_top));
assert(q->is_gc_marked(), "encoding the pointer should preserve the mark");
} else {
// if the object isn't moving we can just set the mark to the default
// mark and handle it specially later on.
q->init_mark();
- assert(!q->is_forwarded(), "should not be forwarded");
+ assert(SlidingForwarding::is_not_forwarded(q), "should not be forwarded");
}
compact_top += size;
@@ -289,7 +290,8 @@ HeapWord* ContiguousSpace::forward(oop q, size_t size,
#if INCLUDE_SERIALGC
-void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) {
+template <bool ALT_FWD>
+void ContiguousSpace::prepare_for_compaction_impl(CompactPoint* cp) {
// Compute the new addresses for the live objects and store it in the mark
// Used by universe::mark_sweep_phase2()
@@ -322,7 +324,7 @@ void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) {
// prefetch beyond cur_obj
Prefetch::write(cur_obj, interval);
size_t size = cast_to_oop(cur_obj)->size();
- compact_top = cp->space->forward(cast_to_oop(cur_obj), size, cp, compact_top);
+ compact_top = cp->space->forward<ALT_FWD>(cast_to_oop(cur_obj), size, cp, compact_top);
cur_obj += size;
end_of_live = cur_obj;
} else {
@@ -338,7 +340,7 @@ void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) {
// we don't have to compact quite as often.
if (cur_obj == compact_top && dead_spacer.insert_deadspace(cur_obj, end)) {
oop obj = cast_to_oop(cur_obj);
- compact_top = cp->space->forward(obj, obj->size(), cp, compact_top);
+ compact_top = cp->space->forward<ALT_FWD>(obj, obj->size(), cp, compact_top);
end_of_live = end;
} else {
// otherwise, it really is a free region.
@@ -369,7 +371,16 @@ void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) {
cp->space->set_compaction_top(compact_top);
}
-void ContiguousSpace::adjust_pointers() {
+void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) {
+ if (UseAltGCForwarding) {
+ prepare_for_compaction_impl<true>(cp);
+ } else {
+ prepare_for_compaction_impl<false>(cp);
+ }
+}
+
+template <bool ALT_FWD>
+void ContiguousSpace::adjust_pointers_impl() {
// Check first is there is any work to do.
if (used() == 0) {
return; // Nothing to do.
@@ -392,7 +403,7 @@ void ContiguousSpace::adjust_pointers() {
if (cur_obj < first_dead || cast_to_oop(cur_obj)->is_gc_marked()) {
// cur_obj is alive
// point all the oops to the new location
- size_t size = MarkSweep::adjust_pointers(cast_to_oop(cur_obj));
+ size_t size = MarkSweep::adjust_pointers<ALT_FWD>(cast_to_oop(cur_obj));
debug_only(prev_obj = cur_obj);
cur_obj += size;
} else {
@@ -406,7 +417,16 @@ void ContiguousSpace::adjust_pointers() {
assert(cur_obj == end_of_live, "just checking");
}
-void ContiguousSpace::compact() {
+void ContiguousSpace::adjust_pointers() {
+ if (UseAltGCForwarding) {
+ adjust_pointers_impl<true>();
+ } else {
+ adjust_pointers_impl<false>();
+ }
+}
+
+template <bool ALT_FWD>
+void ContiguousSpace::compact_impl() {
// Copy all live objects to their new location
// Used by MarkSweep::mark_sweep_phase4()
@@ -435,7 +455,7 @@ void ContiguousSpace::compact() {
debug_only(HeapWord* prev_obj = nullptr);
while (cur_obj < end_of_live) {
- if (!cast_to_oop(cur_obj)->is_forwarded()) {
+ if (SlidingForwarding::is_not_forwarded(cast_to_oop(cur_obj))) {
debug_only(prev_obj = cur_obj);
// The first word of the dead object contains a pointer to the next live object or end of space.
cur_obj = *(HeapWord**)cur_obj;
@@ -446,7 +466,7 @@ void ContiguousSpace::compact() {
// size and destination
size_t size = cast_to_oop(cur_obj)->size();
- HeapWord* compaction_top = cast_from_oop<HeapWord*>(cast_to_oop(cur_obj)->forwardee());
+ HeapWord* compaction_top = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<ALT_FWD>(cast_to_oop(cur_obj)));
// prefetch beyond compaction_top
Prefetch::write(compaction_top, copy_interval);
@@ -469,6 +489,14 @@ void ContiguousSpace::compact() {
clear_empty_region(this);
}
+void ContiguousSpace::compact() {
+ if (UseAltGCForwarding) {
+ compact_impl<true>();
+ } else {
+ compact_impl<false>();
+ }
+}
+
#endif // INCLUDE_SERIALGC
void Space::print_short() const { print_short_on(tty); }
@@ -311,7 +311,18 @@ private:
static inline void clear_empty_region(ContiguousSpace* space);
- protected:
+#if INCLUDE_SERIALGC
+ template <bool ALT_FWD>
+ void prepare_for_compaction_impl(CompactPoint* cp);
+
+ template <bool ALT_FWD>
+ void adjust_pointers_impl();
+
+ template <bool ALT_FWD>
+ void compact_impl();
+#endif
+
+protected:
HeapWord* _top;
// A helper for mangling the unused area of the space in debug builds.
GenSpaceMangler* _mangler;
@@ -398,7 +409,8 @@ private:
// and then forward. In either case, returns the new value of "compact_top".
// Invokes the "alloc_block" function of the then-current compaction
// space.
- virtual HeapWord* forward(oop q, size_t size, CompactPoint* cp,
+ template <bool ALT_FWD>
+ HeapWord* forward(oop q, size_t size, CompactPoint* cp,
HeapWord* compact_top);
// Accessors
@@ -197,7 +197,7 @@ void ShenandoahAsserts::assert_correct(void* interior_loc, oop obj, const char*
file, line);
}
- Klass* obj_klass = obj->klass_or_null();
+ Klass* obj_klass = obj->forward_safe_klass();
if (obj_klass == nullptr) {
print_failure(_safe_unknown, obj, interior_loc, nullptr, "Shenandoah assert_correct failed",
"Object klass pointer should not be null",
@@ -235,7 +235,7 @@ void ShenandoahAsserts::assert_correct(void* interior_loc, oop obj, const char*
file, line);
}
- if (obj_klass != fwd->klass()) {
+ if (obj_klass != fwd->forward_safe_klass()) {
print_failure(_safe_oop, obj, interior_loc, nullptr, "Shenandoah assert_correct failed",
"Forwardee klass disagrees with object class",
file, line);
@@ -28,6 +28,7 @@
#include "gc/shared/continuationGCSupport.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
#include "gc/shared/tlab_globals.hpp"
#include "gc/shared/workerThread.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
@@ -221,6 +222,8 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
// until all phases run together.
ShenandoahHeapLocker lock(heap->lock());
+ SlidingForwarding::begin();
+
phase2_calculate_target_addresses(worker_slices);
OrderAccess::fence();
@@ -234,6 +237,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
// Epilogue
_preserved_marks->restore(heap->workers());
_preserved_marks->reclaim();
+ SlidingForwarding::end();
}
// Resize metaspace
@@ -295,6 +299,7 @@ void ShenandoahFullGC::phase1_mark_heap() {
heap->parallel_cleaning(true /* full_gc */);
}
+template <bool ALT_FWD>
class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
private:
PreservedMarks* const _preserved_marks;
@@ -364,7 +369,7 @@ public:
shenandoah_assert_not_forwarded(nullptr, p);
if (_compact_point != cast_from_oop<HeapWord*>(p)) {
_preserved_marks->push_if_necessary(p, p->mark());
- p->forward_to(cast_to_oop(_compact_point));
+ SlidingForwarding::forward_to<ALT_FWD>(p, cast_to_oop(_compact_point));
}
_compact_point += obj_size;
}
@@ -396,6 +401,16 @@ public:
}
void work(uint worker_id) {
+ if (UseAltGCForwarding) {
+ work_impl<true>(worker_id);
+ } else {
+ work_impl<false>(worker_id);
+ }
+ }
+
+private:
+ template <bool ALT_FWD>
+ void work_impl(uint worker_id) {
ShenandoahParallelWorkerSession worker_session(worker_id);
ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
ShenandoahHeapRegionSetIterator it(slice);
@@ -411,7 +426,7 @@ public:
GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
- ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
+ ShenandoahPrepareForCompactionObjectClosure<ALT_FWD> cl(_preserved_marks->get(worker_id), empty_regions, from_region);
while (from_region != nullptr) {
assert(is_candidate_region(from_region), "Sanity");
@@ -437,7 +452,8 @@ public:
}
};
-void ShenandoahFullGC::calculate_target_humongous_objects() {
+template <bool ALT_FWD>
+void ShenandoahFullGC::calculate_target_humongous_objects_impl() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
// Compute the new addresses for humongous objects. We need to do this after addresses
@@ -473,7 +489,7 @@ void ShenandoahFullGC::calculate_target_humongous_objects() {
if (start >= to_begin && start != r->index()) {
// Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
_preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark());
- old_obj->forward_to(cast_to_oop(heap->get_region(start)->bottom()));
+ SlidingForwarding::forward_to<ALT_FWD>(old_obj, cast_to_oop(heap->get_region(start)->bottom()));
to_end = start;
continue;
}
@@ -485,6 +501,14 @@ void ShenandoahFullGC::calculate_target_humongous_objects() {
}
}
+void ShenandoahFullGC::calculate_target_humongous_objects() {
+ if (UseAltGCForwarding) {
+ calculate_target_humongous_objects_impl<true>();
+ } else {
+ calculate_target_humongous_objects_impl<false>();
+ }
+}
+
class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
private:
ShenandoahHeap* const _heap;
@@ -722,6 +746,7 @@ void ShenandoahFullGC::phase2_calculate_target_addresses(ShenandoahHeapRegionSet
}
}
+template <bool ALT_FWD>
class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure {
private:
ShenandoahHeap* const _heap;
@@ -733,8 +758,8 @@ private:
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
assert(_ctx->is_marked(obj), "must be marked");
- if (obj->is_forwarded()) {
- oop forw = obj->forwardee();
+ if (SlidingForwarding::is_forwarded(obj)) {
+ oop forw = SlidingForwarding::forwardee<ALT_FWD>(obj);
RawAccess<IS_NOT_NULL>::oop_store(p, forw);
}
}
@@ -751,10 +776,11 @@ public:
void do_nmethod(nmethod* nm) {}
};
+template <bool ALT_FWD>
class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
- ShenandoahAdjustPointersClosure _cl;
+ ShenandoahAdjustPointersClosure<ALT_FWD> _cl;
public:
ShenandoahAdjustPointersObjectClosure() :
@@ -777,9 +803,11 @@ public:
_heap(ShenandoahHeap::heap()) {
}
- void work(uint worker_id) {
+private:
+ template <bool ALT_FWD>
+ void work_impl(uint worker_id) {
ShenandoahParallelWorkerSession worker_session(worker_id);
- ShenandoahAdjustPointersObjectClosure obj_cl;
+ ShenandoahAdjustPointersObjectClosure<ALT_FWD> obj_cl;
ShenandoahHeapRegion* r = _regions.next();
while (r != nullptr) {
if (!r->is_humongous_continuation() && r->has_live()) {
@@ -788,24 +816,45 @@ public:
r = _regions.next();
}
}
+
+public:
+ void work(uint worker_id) {
+ if (UseAltGCForwarding) {
+ work_impl<true>(worker_id);
+ } else {
+ work_impl<false>(worker_id);
+ }
+ }
};
class ShenandoahAdjustRootPointersTask : public WorkerTask {
private:
ShenandoahRootAdjuster* _rp;
PreservedMarksSet* _preserved_marks;
+
public:
ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
WorkerTask("Shenandoah Adjust Root Pointers"),
_rp(rp),
_preserved_marks(preserved_marks) {}
- void work(uint worker_id) {
+private:
+ template <bool ALT_FWD>
+ void work_impl(uint worker_id) {
ShenandoahParallelWorkerSession worker_session(worker_id);
- ShenandoahAdjustPointersClosure cl;
+ ShenandoahAdjustPointersClosure<ALT_FWD> cl;
_rp->roots_do(worker_id, &cl);
_preserved_marks->get(worker_id)->adjust_during_full_gc();
}
+
+public:
+ void work(uint worker_id) {
+ if (UseAltGCForwarding) {
+ work_impl<true>(worker_id);
+ } else {
+ work_impl<false>(worker_id);
+ }
+ }
};
void ShenandoahFullGC::phase3_update_references() {
@@ -832,6 +881,7 @@ void ShenandoahFullGC::phase3_update_references() {
workers->run_task(&adjust_pointers_task);
}
+template <bool ALT_FWD>
class ShenandoahCompactObjectsClosure : public ObjectClosure {
private:
ShenandoahHeap* const _heap;
@@ -844,9 +894,9 @@ public:
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
size_t size = p->size();
- if (p->is_forwarded()) {
+ if (SlidingForwarding::is_forwarded(p)) {
HeapWord* compact_from = cast_from_oop<HeapWord*>(p);
- HeapWord* compact_to = cast_from_oop<HeapWord*>(p->forwardee());
+ HeapWord* compact_to = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<ALT_FWD>(p));
assert(compact_from != compact_to, "Forwarded object should move");
Copy::aligned_conjoint_words(compact_from, compact_to, size);
oop new_obj = cast_to_oop(compact_to);
@@ -869,11 +919,13 @@ public:
_worker_slices(worker_slices) {
}
- void work(uint worker_id) {
+private:
+ template <bool ALT_FWD>
+ void work_impl(uint worker_id) {
ShenandoahParallelWorkerSession worker_session(worker_id);
ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
- ShenandoahCompactObjectsClosure cl(worker_id);
+ ShenandoahCompactObjectsClosure<ALT_FWD> cl(worker_id);
ShenandoahHeapRegion* r = slice.next();
while (r != nullptr) {
assert(!r->is_humongous(), "must not get humongous regions here");
@@ -884,6 +936,15 @@ public:
r = slice.next();
}
}
+
+public:
+ void work(uint worker_id) {
+ if (UseAltGCForwarding) {
+ work_impl<true>(worker_id);
+ } else {
+ work_impl<false>(worker_id);
+ }
+ }
};
class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
@@ -939,7 +1000,8 @@ public:
}
};
-void ShenandoahFullGC::compact_humongous_objects() {
+template <bool ALT_FWD>
+void ShenandoahFullGC::compact_humongous_objects_impl() {
// Compact humongous regions, based on their fwdptr objects.
//
// This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
@@ -952,7 +1014,7 @@ void ShenandoahFullGC::compact_humongous_objects() {
ShenandoahHeapRegion* r = heap->get_region(c - 1);
if (r->is_humongous_start()) {
oop old_obj = cast_to_oop(r->bottom());
- if (!old_obj->is_forwarded()) {
+ if (SlidingForwarding::is_not_forwarded(old_obj)) {
// No need to move the object, it stays at the same slot
continue;
}
@@ -961,7 +1023,7 @@ void ShenandoahFullGC::compact_humongous_objects() {
size_t old_start = r->index();
size_t old_end = old_start + num_regions - 1;
- size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
+ size_t new_start = heap->heap_region_index_containing(SlidingForwarding::forwardee<ALT_FWD>(old_obj));
size_t new_end = new_start + num_regions - 1;
assert(old_start != new_start, "must be real move");
assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index());
@@ -1002,6 +1064,14 @@ void ShenandoahFullGC::compact_humongous_objects() {
}
}
+void ShenandoahFullGC::compact_humongous_objects() {
+ if (UseAltGCForwarding) {
+ compact_humongous_objects_impl<true>();
+ } else {
+ compact_humongous_objects_impl<false>();
+ }
+}
+
// This is slightly different to ShHeap::reset_next_mark_bitmap:
// we need to remain able to walk pinned regions.
// Since pinned region do not move and don't get compacted, we will get holes with
@@ -55,6 +55,7 @@ class VM_ShenandoahFullGC;
class ShenandoahDegenGC;
class ShenandoahFullGC : public ShenandoahGC {
+ template <bool ALT_FWD>
friend class ShenandoahPrepareForCompactionObjectClosure;
friend class VM_ShenandoahFullGC;
friend class ShenandoahDegenGC;
@@ -83,7 +84,11 @@ private:
void phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices);
void distribute_slices(ShenandoahHeapRegionSet** worker_slices);
+ template <bool ALT_FWD>
+ void calculate_target_humongous_objects_impl();
void calculate_target_humongous_objects();
+ template <bool ALT_FWD>
+ void compact_humongous_objects_impl();
void compact_humongous_objects();
};
@@ -34,6 +34,7 @@
#include "gc/shared/locationPrinter.inline.hpp"
#include "gc/shared/memAllocator.hpp"
#include "gc/shared/plab.hpp"
+#include "gc/shared/slidingForwarding.hpp"
#include "gc/shared/tlab_globals.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
@@ -404,6 +405,8 @@ jint ShenandoahHeap::initialize() {
ShenandoahInitLogger::print();
+ SlidingForwarding::initialize(_heap_region, ShenandoahHeapRegion::region_size_words());
+
return JNI_OK;
}
@@ -284,7 +284,7 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
- size_t size = p->size();
+ size_t size = p->forward_safe_size();
assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
@@ -319,11 +319,28 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
// Copy the object:
Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, size);
-
- // Try to install the new forwarding pointer.
oop copy_val = cast_to_oop(copy);
- ContinuationGCSupport::relativize_stack_chunk(copy_val);
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ // The copy above is not atomic. Make sure we have seen the proper mark
+ // and re-install it into the copy, so that Klass* is guaranteed to be correct.
+ markWord mark = copy_val->mark();
+ if (!mark.is_marked()) {
+ copy_val->set_mark(mark);
+ ContinuationGCSupport::relativize_stack_chunk(copy_val);
+ } else {
+ // If we copied a mark-word that indicates 'forwarded' state, the object
+ // installation would not succeed. We cannot access Klass* anymore either.
+ // Skip the transformation.
+ }
+ } else
+#endif
+ {
+ ContinuationGCSupport::relativize_stack_chunk(copy_val);
+ }
+
+ // Try to install the new forwarding pointer.
oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
if (result == copy_val) {
// Successfully evacuated. Our copy is now the public one!
@@ -499,7 +516,7 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region,
oop obj = cast_to_oop(cs);
assert(oopDesc::is_oop(obj), "sanity");
assert(ctx->is_marked(obj), "object expected to be marked");
- size_t size = obj->size();
+ size_t size = obj->forward_safe_size();
cl->do_object(obj);
cs += size;
}
@@ -102,7 +102,7 @@ private:
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
- if (is_instance_ref_klass(obj->klass())) {
+ if (is_instance_ref_klass(obj->forward_safe_klass())) {
obj = ShenandoahForwarding::get_forwardee(obj);
}
// Single threaded verification can use faster non-atomic stack and bitmap
@@ -129,7 +129,7 @@ private:
"oop must be aligned");
ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj);
- Klass* obj_klass = obj->klass_or_null();
+ Klass* obj_klass = obj->forward_safe_klass();
// Verify that obj is not in dead space:
{
@@ -144,11 +144,11 @@ private:
"Object start should be within the region");
if (!obj_reg->is_humongous()) {
- check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(),
+ check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->forward_safe_size()) <= obj_reg->top(),
"Object end should be within the region");
} else {
size_t humongous_start = obj_reg->index();
- size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift());
+ size_t humongous_end = humongous_start + (obj->forward_safe_size() >> ShenandoahHeapRegion::region_size_words_shift());
for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) {
check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(),
"Humongous object is in continuation that fits it");
@@ -165,7 +165,7 @@ private:
// skip
break;
case ShenandoahVerifier::_verify_liveness_complete:
- Atomic::add(&_ld[obj_reg->index()], (uint) obj->size(), memory_order_relaxed);
+ Atomic::add(&_ld[obj_reg->index()], (uint) obj->forward_safe_size(), memory_order_relaxed);
// fallthrough for fast failure for un-live regions:
case ShenandoahVerifier::_verify_liveness_conservative:
check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
@@ -209,7 +209,7 @@ private:
HeapWord *fwd_addr = cast_from_oop<HeapWord *>(fwd);
check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(),
"Forwardee start should be within the region");
- check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
+ check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->forward_safe_size()) <= fwd_reg->top(),
"Forwardee end should be within the region");
oop fwd2 = ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
@@ -312,7 +312,8 @@ public:
*/
void verify_oops_from(oop obj) {
_loc = obj;
- obj->oop_iterate(this);
+ Klass* klass = obj->forward_safe_klass();
+ obj->oop_iterate_backwards(this, klass);
_loc = nullptr;
}
@@ -592,7 +593,7 @@ public:
// Verify everything reachable from that object too, hopefully realizing
// everything was already marked, and never touching further:
- if (!is_instance_ref_klass(obj->klass())) {
+ if (!is_instance_ref_klass(obj->forward_safe_klass())) {
cl.verify_oops_from(obj);
(*processed)++;
}
@@ -298,7 +298,7 @@ void XBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* a
assert(src_offset == dest_offset, "should be equal");
jlong offset = src_offset->get_long();
if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
- assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
+ assert(!UseCompressedClassPointers AARCH64_ONLY(|| UseCompactObjectHeaders), "should only happen without compressed class pointers");
assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
length = phase->transform_later(new SubLNode(length, phase->longcon(1))); // Size is in longs
src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
@@ -50,7 +50,17 @@ oop XObjArrayAllocator::initialize(HeapWord* mem) const {
// time and time-to-safepoint
const size_t segment_max = XUtils::bytes_to_words(64 * K);
const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
- const size_t header = arrayOopDesc::header_size(element_type);
+
+ // Clear leading 32 bits, if necessary.
+ int base_offset = arrayOopDesc::base_offset_in_bytes(element_type);
+ if (!is_aligned(base_offset, HeapWordSize)) {
+ assert(is_aligned(base_offset, BytesPerInt), "array base must be 32 bit aligned");
+ *reinterpret_cast<jint*>(reinterpret_cast<char*>(mem) + base_offset) = 0;
+ base_offset += BytesPerInt;
+ }
+ assert(is_aligned(base_offset, HeapWordSize), "remaining array base must be 64 bit aligned");
+
+ const size_t header = heap_word_size(base_offset);
const size_t payload_size = _word_size - header;
if (payload_size <= segment_max) {
@@ -63,8 +73,15 @@ oop XObjArrayAllocator::initialize(HeapWord* mem) const {
// The array is going to be exposed before it has been completely
// cleared, therefore we can't expose the header at the end of this
// function. Instead explicitly initialize it according to our needs.
- arrayOopDesc::set_mark(mem, markWord::prototype());
- arrayOopDesc::release_set_klass(mem, _klass);
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ arrayOopDesc::release_set_mark(mem, _klass->prototype_header());
+ } else
+#endif
+ {
+ arrayOopDesc::set_mark(mem, markWord::prototype());
+ arrayOopDesc::release_set_klass(mem, _klass);
+ }
assert(_length >= 0, "length should be non-negative");
arrayOopDesc::set_length(mem, _length);
@@ -483,7 +483,7 @@ void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* a
assert(src_offset == dest_offset, "should be equal");
const jlong offset = src_offset->get_long();
if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
- assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
+ assert(!UseCompressedClassPointers AARCH64_ONLY(|| UseCompactObjectHeaders), "should only happen without compressed class pointers");
assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
length = phase->transform_later(new SubLNode(length, phase->longcon(1))); // Size is in longs
src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
@@ -50,7 +50,17 @@ oop ZObjArrayAllocator::initialize(HeapWord* mem) const {
// time and time-to-safepoint
const size_t segment_max = ZUtils::bytes_to_words(64 * K);
const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
- const size_t header = arrayOopDesc::header_size(element_type);
+
+ // Clear leading 32 bits, if necessary.
+ int base_offset = arrayOopDesc::base_offset_in_bytes(element_type);
+ if (!is_aligned(base_offset, HeapWordSize)) {
+ assert(is_aligned(base_offset, BytesPerInt), "array base must be 32 bit aligned");
+ *reinterpret_cast<jint*>(reinterpret_cast<char*>(mem) + base_offset) = 0;
+ base_offset += BytesPerInt;
+ }
+ assert(is_aligned(base_offset, HeapWordSize), "remaining array base must be 64 bit aligned");
+
+ const size_t header = heap_word_size(base_offset);
const size_t payload_size = _word_size - header;
if (payload_size <= segment_max) {
@@ -66,8 +76,15 @@ oop ZObjArrayAllocator::initialize(HeapWord* mem) const {
// Signal to the ZIterator that this is an invisible root, by setting
// the mark word to "marked". Reset to prototype() after the clearing.
- arrayOopDesc::set_mark(mem, markWord::prototype().set_marked());
- arrayOopDesc::release_set_klass(mem, _klass);
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ arrayOopDesc::release_set_mark(mem, _klass->prototype_header().set_marked());
+ } else
+#endif
+ {
+ arrayOopDesc::set_mark(mem, markWord::prototype().set_marked());
+ arrayOopDesc::release_set_klass(mem, _klass);
+ }
assert(_length >= 0, "length should be non-negative");
arrayOopDesc::set_length(mem, _length);
@@ -135,7 +152,14 @@ oop ZObjArrayAllocator::initialize(HeapWord* mem) const {
ZThreadLocalData::clear_invisible_root(_thread);
// Signal to the ZIterator that this is no longer an invisible root
- oopDesc::release_set_mark(mem, markWord::prototype());
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ oopDesc::release_set_mark(mem, _klass->prototype_header());
+ } else
+#endif
+ {
+ oopDesc::release_set_mark(mem, markWord::prototype());
+ }
return cast_to_oop(mem);
}
@@ -621,7 +621,6 @@ private:
zaddress try_relocate_object_inner(zaddress from_addr) {
ZForwardingCursor cursor;
- const size_t size = ZUtils::object_size(from_addr);
ZPage* const to_page = target(_forwarding->to_age());
// Lookup forwarding
@@ -629,12 +628,14 @@ private:
const zaddress to_addr = forwarding_find(_forwarding, from_addr, &cursor);
if (!is_null(to_addr)) {
// Already relocated
+ const size_t size = ZUtils::object_size(to_addr);
increase_other_forwarded(size);
return to_addr;
}
}
// Allocate object
+ const size_t size = ZUtils::object_size(from_addr);
const zaddress allocated_addr = _allocator->alloc_object(to_page, size);
if (is_null(allocated_addr)) {
// Allocation failed
@@ -1997,10 +1997,13 @@ run:
}
// Initialize header, mirrors MemAllocator.
- oopDesc::set_mark(result, markWord::prototype());
- oopDesc::set_klass_gap(result, 0);
- oopDesc::release_set_klass(result, ik);
-
+ if (UseCompactObjectHeaders) {
+ oopDesc::release_set_mark(result, ik->prototype_header());
+ } else {
+ oopDesc::set_mark(result, markWord::prototype());
+ oopDesc::set_klass_gap(result, 0);
+ oopDesc::release_set_klass(result, ik);
+ }
oop obj = cast_to_oop(result);
// Must prevent reordering of stores for object initialization
@@ -70,7 +70,7 @@ class ObjectSampleMarker : public StackObj {
// now we will set the mark word to "marked" in order to quickly
// identify sample objects during the reachability search from gc roots.
assert(!obj->mark().is_marked(), "should only mark an object once");
- obj->set_mark(markWord::prototype().set_marked());
+ obj->set_mark(obj->prototype_mark().set_marked());
assert(obj->mark().is_marked(), "invariant");
}
};
@@ -2388,7 +2388,7 @@ C2V_END
C2V_VMENTRY_0(jint, arrayBaseOffset, (JNIEnv* env, jobject, jchar type_char))
BasicType type = JVMCIENV->typeCharToBasicType(type_char, JVMCI_CHECK_0);
- return arrayOopDesc::header_size(type) * HeapWordSize;
+ return arrayOopDesc::base_offset_in_bytes(type);
C2V_END
C2V_VMENTRY_0(jint, arrayIndexScale, (JNIEnv* env, jobject, jchar type_char))
@@ -275,7 +275,6 @@ JVMCIObjectArray CompilerToVM::initialize_intrinsics(JVMCI_TRAPS) {
do_bool_flag(Inline) \
do_intx_flag(JVMCICounterSize) \
do_bool_flag(JVMCIPrintProperties) \
- do_bool_flag(JVMCIUseFastLocking) \
do_int_flag(ObjectAlignmentInBytes) \
do_bool_flag(PrintInlining) \
do_bool_flag(ReduceInitialCardMarks) \
@@ -123,7 +123,6 @@ bool JVMCIGlobals::check_jvmci_flags_are_consistent() {
CHECK_NOT_SET(JVMCITraceLevel, EnableJVMCI)
CHECK_NOT_SET(JVMCICounterSize, EnableJVMCI)
CHECK_NOT_SET(JVMCICountersExcludeCompiler, EnableJVMCI)
- CHECK_NOT_SET(JVMCIUseFastLocking, EnableJVMCI)
CHECK_NOT_SET(JVMCINMethodSizeLimit, EnableJVMCI)
CHECK_NOT_SET(JVMCIPrintProperties, EnableJVMCI)
CHECK_NOT_SET(JVMCIThreadsPerNativeLibraryRuntime, EnableJVMCI)
@@ -127,9 +127,6 @@ class fileStream;
product(bool, JVMCICountersExcludeCompiler, true, EXPERIMENTAL, \
"Exclude JVMCI compiler threads from benchmark counters") \
\
- develop(bool, JVMCIUseFastLocking, true, \
- "Use fast inlined locking code") \
- \
product(intx, JVMCINMethodSizeLimit, (80*K)*wordSize, EXPERIMENTAL, \
"Maximum size of a compiled method.") \
range(0, max_jint) \
@@ -216,9 +216,12 @@
nonstatic_field(JavaThread, _poll_data, SafepointMechanism::ThreadData) \
nonstatic_field(JavaThread, _stack_overflow_state._reserved_stack_activation, address) \
nonstatic_field(JavaThread, _held_monitor_count, int64_t) \
+ nonstatic_field(JavaThread, _lock_stack, LockStack) \
JVMTI_ONLY(nonstatic_field(JavaThread, _is_in_VTMS_transition, bool)) \
JVMTI_ONLY(nonstatic_field(JavaThread, _is_in_tmp_VTMS_transition, bool)) \
\
+ nonstatic_field(LockStack, _top, uint32_t) \
+ \
JVMTI_ONLY(static_field(JvmtiVTMSTransitionDisabler, _VTMS_notify_jvmti_events, bool)) \
\
static_field(java_lang_Class, _klass_offset, int) \
@@ -493,6 +496,7 @@
declare_constant(BranchData::not_taken_off_set) \
\
declare_constant_with_value("CardTable::dirty_card", CardTable::dirty_card_val()) \
+ declare_constant_with_value("LockStack::_end_offset", LockStack::end_offset()) \
\
declare_constant(CodeInstaller::VERIFIED_ENTRY) \
declare_constant(CodeInstaller::UNVERIFIED_ENTRY) \
@@ -677,6 +681,10 @@
declare_constant(InstanceKlass::being_initialized) \
declare_constant(InstanceKlass::fully_initialized) \
\
+ declare_constant(LockingMode::LM_MONITOR) \
+ declare_constant(LockingMode::LM_LEGACY) \
+ declare_constant(LockingMode::LM_LIGHTWEIGHT) \
+ \
/*********************************/ \
/* InstanceKlass _misc_flags */ \
/*********************************/ \
@@ -724,6 +732,8 @@
AARCH64_ONLY(declare_constant(NMethodPatchingType::conc_instruction_and_data_patch)) \
AARCH64_ONLY(declare_constant(NMethodPatchingType::conc_data_patch)) \
\
+ declare_constant(ObjectMonitor::ANONYMOUS_OWNER) \
+ \
declare_constant(ReceiverTypeData::nonprofiled_count_off_set) \
declare_constant(ReceiverTypeData::receiver_type_row_cell_count) \
declare_constant(ReceiverTypeData::receiver0_offset) \
@@ -325,8 +325,16 @@ void Universe::genesis(TRAPS) {
HandleMark hm(THREAD);
// Explicit null checks are needed if these offsets are not smaller than the page size
- assert(oopDesc::klass_offset_in_bytes() < static_cast<intptr_t>(os::vm_page_size()),
- "Klass offset is expected to be less than the page size");
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ assert(oopDesc::mark_offset_in_bytes() < static_cast<intptr_t>(os::vm_page_size()),
+ "Mark offset is expected to be less than the page size");
+ } else
+#endif
+ {
+ assert(oopDesc::klass_offset_in_bytes() < static_cast<intptr_t>(os::vm_page_size()),
+ "Klass offset is expected to be less than the page size");
+ }
assert(arrayOopDesc::length_offset_in_bytes() < static_cast<intptr_t>(os::vm_page_size()),
"Array length offset is expected to be less than the page size");
@@ -27,6 +27,7 @@
#include "oops/oop.hpp"
#include "utilities/align.hpp"
+#include "utilities/globalDefinitions.hpp"
// arrayOopDesc is the abstract baseclass for all arrays. It doesn't
// declare pure virtual to enforce this because that would allocate a vtbl
@@ -45,47 +46,60 @@ class arrayOopDesc : public oopDesc {
// Interpreter/Compiler offsets
- // Header size computation.
- // The header is considered the oop part of this type plus the length.
- // Returns the aligned header_size_in_bytes. This is not equivalent to
- // sizeof(arrayOopDesc) which should not appear in the code.
- static int header_size_in_bytes() {
- size_t hs = align_up(length_offset_in_bytes() + sizeof(int),
- HeapWordSize);
-#ifdef ASSERT
- // make sure it isn't called before UseCompressedOops is initialized.
- static size_t arrayoopdesc_hs = 0;
- if (arrayoopdesc_hs == 0) arrayoopdesc_hs = hs;
- assert(arrayoopdesc_hs == hs, "header size can't change");
-#endif // ASSERT
- return (int)hs;
- }
-
+private:
// Returns the address of the length "field". See length_offset_in_bytes().
static int* length_addr_impl(void* obj_ptr) {
char* ptr = static_cast<char*>(obj_ptr);
return reinterpret_cast<int*>(ptr + length_offset_in_bytes());
}
- // Check whether an element of a typeArrayOop with the given type must be
- // aligned 0 mod 8. The typeArrayOop itself must be aligned at least this
+ // Check whether an element of an arrayOop with the given type must be
+ // aligned 0 mod 8. The arrayOop itself must be aligned at least this
// strongly.
static bool element_type_should_be_aligned(BasicType type) {
+#ifdef _LP64
+ if (type == T_OBJECT || type == T_ARRAY) {
+ return !UseCompressedOops;
+ }
+#endif
return type == T_DOUBLE || type == T_LONG;
}
public:
+ // Header size computation.
+ // The header is considered the oop part of this type plus the length.
+ // This is not equivalent to sizeof(arrayOopDesc) which should not appear in the code.
+ static int header_size_in_bytes() {
+ size_t hs = length_offset_in_bytes() + sizeof(int);
+#ifdef ASSERT
+ // make sure it isn't called before UseCompressedOops is initialized.
+ static size_t arrayoopdesc_hs = 0;
+ if (arrayoopdesc_hs == 0) arrayoopdesc_hs = hs;
+ assert(arrayoopdesc_hs == hs, "header size can't change");
+#endif // ASSERT
+ return (int)hs;
+ }
+
// The _length field is not declared in C++. It is allocated after the
// declared nonstatic fields in arrayOopDesc if not compressed, otherwise
// it occupies the second half of the _klass field in oopDesc.
static int length_offset_in_bytes() {
- return UseCompressedClassPointers ? klass_gap_offset_in_bytes() :
- sizeof(arrayOopDesc);
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ return oopDesc::base_offset_in_bytes();
+ } else
+#endif
+ if (UseCompressedClassPointers) {
+ return klass_gap_offset_in_bytes();
+ } else {
+ return sizeof(arrayOopDesc);
+ }
}
// Returns the offset of the first element.
static int base_offset_in_bytes(BasicType type) {
- return header_size(type) * HeapWordSize;
+ size_t hs = header_size_in_bytes();
+ return (int)(element_type_should_be_aligned(type) ? align_up(hs, BytesPerLong) : hs);
}
// Returns the address of the first element. The elements in the array will not
@@ -122,18 +136,7 @@ class arrayOopDesc : public oopDesc {
*length_addr_impl(mem) = length;
}
- // Should only be called with constants as argument
- // (will not constant fold otherwise)
- // Returns the header size in words aligned to the requirements of the
- // array object type.
- static int header_size(BasicType type) {
- size_t typesize_in_bytes = header_size_in_bytes();
- return (int)(element_type_should_be_aligned(type)
- ? align_object_offset(typesize_in_bytes/HeapWordSize)
- : typesize_in_bytes/HeapWordSize);
- }
-
- // Return the maximum length of an array of BasicType. The length can passed
+ // Return the maximum length of an array of BasicType. The length can be passed
// to typeArrayOop::object_size(scale, length, header_size) without causing an
// overflow. We also need to make sure that this will not overflow a size_t on
// 32 bit platforms when we convert it to a byte size.
@@ -141,8 +144,12 @@ class arrayOopDesc : public oopDesc {
assert(type >= 0 && type < T_CONFLICT, "wrong type");
assert(type2aelembytes(type) != 0, "wrong type");
+ size_t hdr_size_in_bytes = base_offset_in_bytes(type);
+ // This is rounded-up and may overlap with the first array elements.
+ size_t hdr_size_in_words = align_up(hdr_size_in_bytes, HeapWordSize) / HeapWordSize;
+
const size_t max_element_words_per_size_t =
- align_down((SIZE_MAX/HeapWordSize - header_size(type)), MinObjAlignment);
+ align_down((SIZE_MAX/HeapWordSize - hdr_size_in_words), MinObjAlignment);
const size_t max_elements_per_size_t =
HeapWordSize * max_element_words_per_size_t / type2aelembytes(type);
if ((size_t)max_jint < max_elements_per_size_t) {
@@ -150,7 +157,7 @@ class arrayOopDesc : public oopDesc {
// (CollectedHeap, Klass::oop_oop_iterate(), and more) uses an int for
// passing around the size (in words) of an object. So, we need to avoid
// overflowing an int when we add the header. See CRs 4718400 and 7110613.
- return align_down(max_jint - header_size(type), MinObjAlignment);
+ return align_down(max_jint - hdr_size_in_words, MinObjAlignment);
}
return (int32_t)max_elements_per_size_t;
}
@@ -33,15 +33,18 @@
class instanceOopDesc : public oopDesc {
public:
- // aligned header size.
- static int header_size() { return sizeof(instanceOopDesc)/HeapWordSize; }
-
// If compressed, the offset of the fields of the instance may not be aligned.
static int base_offset_in_bytes() {
- return (UseCompressedClassPointers) ?
- klass_gap_offset_in_bytes() :
- sizeof(instanceOopDesc);
-
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ return oopDesc::base_offset_in_bytes();
+ } else
+#endif
+ if (UseCompressedClassPointers) {
+ return klass_gap_offset_in_bytes();
+ } else {
+ return sizeof(instanceOopDesc);
+ }
}
};
@@ -251,11 +251,22 @@ void* Klass::operator new(size_t size, ClassLoaderData* loader_data, size_t word
return Metaspace::allocate(loader_data, word_size, MetaspaceObj::ClassType, THREAD);
}
+static markWord make_prototype(Klass* kls) {
+ markWord prototype = markWord::prototype();
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ prototype = prototype.set_klass(kls);
+ }
+#endif
+ return prototype;
+}
+
// "Normal" instantiation is preceded by a MetaspaceObj allocation
// which zeros out memory - calloc equivalent.
// The constructor is also used from CppVtableCloner,
// which doesn't zero out the memory before calling the constructor.
Klass::Klass(KlassKind kind) : _kind(kind),
+ _prototype_header(make_prototype(this)),
_shared_class_path_index(-1) {
CDS_ONLY(_shared_class_flags = 0;)
CDS_JAVA_HEAP_ONLY(_archived_mirror_index = -1;)
@@ -960,6 +971,12 @@ void Klass::oop_print_on(oop obj, outputStream* st) {
// print header
obj->mark().print_on(st);
st->cr();
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ st->print(BULLET"prototype_header: " INTPTR_FORMAT, _prototype_header.value());
+ st->cr();
+ }
+#endif
}
// print class
@@ -171,6 +171,8 @@ class Klass : public Metadata {
// contention that may happen when a nearby object is modified.
AccessFlags _access_flags; // Access flags. The class/interface distinction is stored here.
+ markWord _prototype_header; // Used to initialize objects' header
+
JFR_ONLY(DEFINE_TRACE_ID_FIELD;)
private:
@@ -705,6 +707,15 @@ protected:
bool is_cloneable() const;
void set_is_cloneable();
+#ifdef AARCH64
+ markWord prototype_header() const {
+ assert(UseCompactObjectHeaders, "only use with compact object headers");
+ return _prototype_header;
+ }
+ inline void set_prototype_header(markWord header);
+#endif
+ static ByteSize prototype_header_offset() { return in_ByteSize(offset_of(Klass, _prototype_header)); }
+
JFR_ONLY(DEFINE_TRACE_ID_METHODS;)
virtual void metaspace_pointers_do(MetaspaceClosure* iter);
@@ -52,6 +52,13 @@ inline bool Klass::is_loader_alive() const {
return class_loader_data()->is_alive();
}
+#ifdef AARCH64
+inline void Klass::set_prototype_header(markWord header) {
+ assert(UseCompactObjectHeaders, "only with compact headers");
+ _prototype_header = header;
+}
+#endif
+
inline oop Klass::java_mirror() const {
return _java_mirror.resolve();
}
@@ -25,6 +25,7 @@
#ifndef SHARE_OOPS_MARKWORD_HPP
#define SHARE_OOPS_MARKWORD_HPP
+#include "gc/shared/gc_globals.hpp"
#include "metaprogramming/primitiveConversions.hpp"
#include "oops/oopsHierarchy.hpp"
#include "runtime/globals.hpp"
@@ -43,6 +44,10 @@
// --------
// unused:25 hash:31 -->| unused_gap:1 age:4 unused_gap:1 lock:2 (normal object)
//
+// 64 bits (with compact headers):
+// -------------------------------
+// nklass:32 hash:25 -->| unused_gap:1 age:4 self-fwded:1 lock:2 (normal object)
+//
// - hash contains the identity hash value: largest value is
// 31 bits, see os::random(). Also, 64-bit vm's require
// a hash value no bigger than 32 bits because they will not
@@ -103,21 +108,43 @@ class markWord {
// Constants
static const int age_bits = 4;
static const int lock_bits = 2;
- static const int first_unused_gap_bits = 1;
- static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - first_unused_gap_bits;
+ static const int self_forwarded_bits = 1;
+ static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - self_forwarded_bits;
static const int hash_bits = max_hash_bits > 31 ? 31 : max_hash_bits;
- static const int second_unused_gap_bits = LP64_ONLY(1) NOT_LP64(0);
+ static const int hash_bits_compact = max_hash_bits > 25 ? 25 : max_hash_bits;
+ // Used only without compact headers.
+ static const int unused_gap_bits = LP64_ONLY(1) NOT_LP64(0);
+#ifdef AARCH64
+ // Used only with compact headers.
+ static const int klass_bits = 32;
+#endif
static const int lock_shift = 0;
- static const int age_shift = lock_bits + first_unused_gap_bits;
- static const int hash_shift = age_shift + age_bits + second_unused_gap_bits;
+ static const int self_forwarded_shift = lock_shift + lock_bits;
+ static const int age_shift = self_forwarded_shift + self_forwarded_bits;
+ static const int hash_shift = age_shift + age_bits + unused_gap_bits;
+ static const int hash_shift_compact = age_shift + age_bits;
+#ifdef AARCH64
+ // Used only with compact headers.
+ static const int klass_shift = hash_shift_compact + hash_bits_compact;
+#endif
static const uintptr_t lock_mask = right_n_bits(lock_bits);
static const uintptr_t lock_mask_in_place = lock_mask << lock_shift;
+ static const uintptr_t self_forwarded_mask = right_n_bits(self_forwarded_bits);
+ static const uintptr_t self_forwarded_mask_in_place = self_forwarded_mask << self_forwarded_shift;
static const uintptr_t age_mask = right_n_bits(age_bits);
static const uintptr_t age_mask_in_place = age_mask << age_shift;
static const uintptr_t hash_mask = right_n_bits(hash_bits);
static const uintptr_t hash_mask_in_place = hash_mask << hash_shift;
+ static const uintptr_t hash_mask_compact = right_n_bits(hash_bits_compact);
+ static const uintptr_t hash_mask_compact_in_place = hash_mask_compact << hash_shift_compact;
+#ifdef AARCH64
+ // Used only with compact headers.
+ static const uintptr_t klass_mask = right_n_bits(klass_bits);
+ static const uintptr_t klass_mask_in_place = klass_mask << klass_shift;
+#endif
+
static const uintptr_t locked_value = 0;
static const uintptr_t unlocked_value = 1;
@@ -205,9 +232,19 @@ class markWord {
markWord displaced_mark_helper() const;
void set_displaced_mark_helper(markWord m) const;
markWord copy_set_hash(intptr_t hash) const {
- uintptr_t tmp = value() & (~hash_mask_in_place);
- tmp |= ((hash & hash_mask) << hash_shift);
- return markWord(tmp);
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ uintptr_t tmp = value() & (~hash_mask_compact_in_place);
+ tmp |= ((hash & hash_mask_compact) << hash_shift_compact);
+ return markWord(tmp);
+ } else {
+#endif
+ uintptr_t tmp = value() & (~hash_mask_in_place);
+ tmp |= ((hash & hash_mask) << hash_shift);
+ return markWord(tmp);
+#ifdef AARCH64
+ }
+#endif
}
// it is only used to be stored into BasicLock as the
// indicator that the lock is using heavyweight monitor
@@ -240,13 +277,30 @@ class markWord {
// hash operations
intptr_t hash() const {
- return mask_bits(value() >> hash_shift, hash_mask);
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ return mask_bits(value() >> hash_shift_compact, hash_mask_compact);
+ } else {
+#endif
+ return mask_bits(value() >> hash_shift, hash_mask);
+#ifdef AARCH64
+ }
+#endif
}
bool has_no_hash() const {
return hash() == no_hash;
}
+#ifdef AARCH64
+ inline markWord actual_mark() const;
+ inline Klass* klass() const;
+ inline Klass* klass_or_null() const;
+ inline narrowKlass narrow_klass() const;
+ inline markWord set_narrow_klass(narrowKlass nklass) const;
+ inline markWord set_klass(Klass* klass) const;
+#endif
+
// Prototype mark for initialization
static markWord prototype() {
return markWord( no_hash_in_place | no_lock_in_place );
@@ -260,6 +314,19 @@ class markWord {
// Recover address of oop from encoded form used in mark
inline void* decode_pointer() { return (void*)clear_lock_bits().value(); }
+
+#ifdef _LP64
+ inline bool self_forwarded() const {
+ bool self_fwd = mask_bits(value(), self_forwarded_mask_in_place) != 0;
+ assert(!self_fwd || UseAltGCForwarding, "Only set self-fwd bit when using alt GC forwarding");
+ return self_fwd;
+ }
+
+ inline markWord set_self_forwarded() const {
+ assert(UseAltGCForwarding, "Only call this with alt GC forwarding");
+ return markWord(value() | self_forwarded_mask_in_place | marked_value);
+ }
+#endif
};
// Support atomic operations.
new file mode 100644
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_OOPS_MARKWORD_INLINE_HPP
+#define SHARE_OOPS_MARKWORD_INLINE_HPP
+
+#include "oops/markWord.hpp"
+#include "oops/compressedOops.inline.hpp"
+
+#ifdef AARCH64
+markWord markWord::actual_mark() const {
+ assert(UseCompactObjectHeaders, "only safe when using compact headers");
+ if (has_displaced_mark_helper()) {
+ return displaced_mark_helper();
+ } else {
+ return *this;
+ }
+}
+
+Klass* markWord::klass() const {
+ assert(UseCompactObjectHeaders, "only used with compact object headers");
+ assert(!CompressedKlassPointers::is_null(narrow_klass()), "narrow klass must not be null: " INTPTR_FORMAT, value());
+ return CompressedKlassPointers::decode_not_null(narrow_klass());
+}
+
+Klass* markWord::klass_or_null() const {
+ assert(UseCompactObjectHeaders, "only used with compact object headers");
+ return CompressedKlassPointers::decode(narrow_klass());
+}
+
+narrowKlass markWord::narrow_klass() const {
+ assert(UseCompactObjectHeaders, "only used with compact object headers");
+ return narrowKlass(value() >> klass_shift);
+}
+
+markWord markWord::set_narrow_klass(narrowKlass nklass) const {
+ assert(UseCompactObjectHeaders, "only used with compact object headers");
+ return markWord((value() & ~klass_mask_in_place) | ((uintptr_t) nklass << klass_shift));
+}
+
+markWord markWord::set_klass(Klass* klass) const {
+ assert(UseCompactObjectHeaders, "only used with compact object headers");
+ assert(UseCompressedClassPointers, "expect compressed klass pointers");
+ narrowKlass nklass = CompressedKlassPointers::encode(const_cast<Klass*>(klass));
+ return set_narrow_klass(nklass);
+}
+#endif // AARCH64
+
+#endif // SHARE_OOPS_MARKWORD_INLINE_HPP
@@ -156,7 +156,8 @@ ObjArrayKlass::ObjArrayKlass(int n, Klass* element_klass, Symbol* name) : ArrayK
}
size_t ObjArrayKlass::oop_size(oop obj) const {
- assert(obj->is_objArray(), "must be object array");
+ // In this assert, we cannot safely access the Klass* with compact headers.
+ assert(AARCH64_ONLY(UseCompactObjectHeaders ||) obj->is_objArray(), "must be object array");
return objArrayOop(obj)->object_size();
}
@@ -70,7 +70,8 @@ void ObjArrayKlass::oop_oop_iterate_elements_bounded(
template <typename T, typename OopClosureType>
void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure) {
- assert (obj->is_array(), "obj must be array");
+ // In this assert, we cannot safely access the Klass* with compact headers.
+ assert (AARCH64_ONLY(UseCompactObjectHeaders ||) obj->is_array(), "obj must be array");
objArrayOop a = objArrayOop(obj);
if (Devirtualizer::do_metadata(closure)) {
@@ -51,32 +51,6 @@ class objArrayOopDesc : public arrayOopDesc {
return base_offset_in_bytes() + sizeof(T) * index;
}
-private:
- // Give size of objArrayOop in HeapWords minus the header
- static int array_size(int length) {
- const uint OopsPerHeapWord = HeapWordSize/heapOopSize;
- assert(OopsPerHeapWord >= 1 && (HeapWordSize % heapOopSize == 0),
- "Else the following (new) computation would be in error");
- uint res = ((uint)length + OopsPerHeapWord - 1)/OopsPerHeapWord;
-#ifdef ASSERT
- // The old code is left in for sanity-checking; it'll
- // go away pretty soon. XXX
- // Without UseCompressedOops, this is simply:
- // oop->length() * HeapWordsPerOop;
- // With narrowOops, HeapWordsPerOop is 1/2 or equal 0 as an integer.
- // The oop elements are aligned up to wordSize
- const uint HeapWordsPerOop = heapOopSize/HeapWordSize;
- uint old_res;
- if (HeapWordsPerOop > 0) {
- old_res = length * HeapWordsPerOop;
- } else {
- old_res = align_up((uint)length, OopsPerHeapWord)/OopsPerHeapWord;
- }
- assert(res == old_res, "Inconsistency between old and new.");
-#endif // ASSERT
- return res;
- }
-
public:
// Returns the offset of the first element.
static int base_offset_in_bytes() {
@@ -94,16 +68,15 @@ private:
oop replace_if_null(int index, oop exchange_value);
// Sizing
- static int header_size() { return arrayOopDesc::header_size(T_OBJECT); }
size_t object_size() { return object_size(length()); }
static size_t object_size(int length) {
// This returns the object size in HeapWords.
- uint asz = array_size(length);
- uint osz = align_object_size(header_size() + asz);
- assert(osz >= asz, "no overflow");
- assert((int)osz > 0, "no overflow");
- return (size_t)osz;
+ size_t asz = (size_t)length * heapOopSize;
+ size_t size_words = heap_word_size(base_offset_in_bytes() + asz);
+ size_t osz = align_object_size(size_words);
+ assert(osz < max_jint, "no overflow");
+ return osz;
}
Klass* element_klass();
@@ -156,7 +156,8 @@ bool oopDesc::is_typeArray_noinline() const { return is_typeArray(); }
bool oopDesc::has_klass_gap() {
// Only has a klass gap when compressed class pointers are used.
- return UseCompressedClassPointers;
+ // Except when using compact headers.
+ return UseCompressedClassPointers AARCH64_ONLY(&& !UseCompactObjectHeaders);
}
#if INCLUDE_CDS_JAVA_HEAP
@@ -168,6 +169,11 @@ void oopDesc::set_narrow_klass(narrowKlass nk) {
#endif
void* oopDesc::load_klass_raw(oop obj) {
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ return obj->klass();
+ } else
+#endif
if (UseCompressedClassPointers) {
narrowKlass narrow_klass = obj->_metadata._compressed_klass;
if (narrow_klass == 0) return nullptr;
@@ -79,6 +79,11 @@ class oopDesc {
inline markWord cas_set_mark(markWord new_mark, markWord old_mark);
inline markWord cas_set_mark(markWord new_mark, markWord old_mark, atomic_memory_order order);
+ inline markWord resolve_mark() const;
+
+ // Returns the prototype mark that should be used for this object.
+ inline markWord prototype_mark() const;
+
// Used only to re-initialize the mark word (e.g., of promoted
// objects during a GC) -- requires a valid klass pointer
inline void init_mark();
@@ -97,7 +102,17 @@ class oopDesc {
static inline void set_klass_gap(HeapWord* mem, int z);
// size of object header, aligned to platform wordSize
- static constexpr int header_size() { return sizeof(oopDesc)/HeapWordSize; }
+ static int header_size() {
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ return sizeof(markWord) / HeapWordSize;
+ } else {
+#endif
+ return sizeof(oopDesc)/HeapWordSize;
+#ifdef AARCH64
+ }
+#endif
+ }
// Returns whether this is an instance of k or an instance of a subclass of k
inline bool is_a(Klass* k) const;
@@ -109,6 +124,20 @@ class oopDesc {
// to be able to figure out the size of an object knowing its klass.
inline size_t size_given_klass(Klass* klass);
+ // The following set of methods is used to access the mark-word and related
+ // properties when the object may be forwarded. Be careful where and when
+ // using this method. It assumes that the forwardee is installed in
+ // the header as a plain pointer (or self-forwarded). In particular,
+ // those methods can not deal with the sliding-forwarding that is used
+ // in Serial, G1 and Shenandoah full-GCs.
+private:
+ inline Klass* forward_safe_klass_impl(markWord m) const;
+public:
+ inline Klass* forward_safe_klass() const;
+ inline Klass* forward_safe_klass(markWord m) const;
+ inline size_t forward_safe_size();
+ inline void forward_safe_init_mark();
+
// type test operations (inlined in oop.inline.hpp)
inline bool is_instance() const;
inline bool is_instanceRef() const;
@@ -261,14 +290,17 @@ class oopDesc {
inline bool is_forwarded() const;
inline void forward_to(oop p);
+ inline void forward_to_self();
// Like "forward_to", but inserts the forwarding pointer atomically.
// Exactly one thread succeeds in inserting the forwarding pointer, and
// this call returns null for that thread; any other thread has the
// value of the forwarding pointer returned and does not modify "this".
inline oop forward_to_atomic(oop p, markWord compare, atomic_memory_order order = memory_order_conservative);
+ inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative);
inline oop forwardee() const;
+ inline oop forwardee(markWord header) const;
// Age of object during scavenge
inline uint age() const;
@@ -312,12 +344,44 @@ class oopDesc {
// for code generation
static int mark_offset_in_bytes() { return (int)offset_of(oopDesc, _mark); }
- static int klass_offset_in_bytes() { return (int)offset_of(oopDesc, _metadata._klass); }
+ static int klass_offset_in_bytes() {
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ STATIC_ASSERT(markWord::klass_shift % 8 == 0);
+ return mark_offset_in_bytes() + markWord::klass_shift / 8;
+ } else
+#endif
+ {
+ return (int)offset_of(oopDesc, _metadata._klass);
+ }
+ }
static int klass_gap_offset_in_bytes() {
assert(has_klass_gap(), "only applicable to compressed klass pointers");
+#ifdef AARCH64
+ assert(!UseCompactObjectHeaders, "don't use klass_offset_in_bytes() with compact headers");
+#endif
return klass_offset_in_bytes() + sizeof(narrowKlass);
}
+ static int base_offset_in_bytes() {
+#ifdef _LP64
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ // With compact headers, the Klass* field is not used for the Klass*
+ // and is used for the object fields instead.
+ STATIC_ASSERT(sizeof(markWord) == 8);
+ return sizeof(markWord);
+ } else
+#endif // AARCH64
+ if (UseCompressedClassPointers) {
+ return sizeof(markWord) + sizeof(narrowKlass);
+ } else
+#endif // _LP64
+ {
+ return sizeof(oopDesc);
+ }
+ }
+
// for error reporting
static void* load_klass_raw(oop obj);
static void* load_oop_raw(oop obj, int offset);
@@ -34,10 +34,11 @@
#include "oops/arrayOop.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/instanceKlass.hpp"
-#include "oops/markWord.hpp"
+#include "oops/markWord.inline.hpp"
#include "oops/oopsHierarchy.hpp"
#include "runtime/atomic.hpp"
#include "runtime/globals.hpp"
+#include "runtime/safepoint.hpp"
#include "utilities/align.hpp"
#include "utilities/debug.hpp"
#include "utilities/macros.hpp"
@@ -66,14 +67,14 @@ void oopDesc::set_mark(HeapWord* mem, markWord m) {
*(markWord*)(((char*)mem) + mark_offset_in_bytes()) = m;
}
-void oopDesc::release_set_mark(HeapWord* mem, markWord m) {
- Atomic::release_store((markWord*)(((char*)mem) + mark_offset_in_bytes()), m);
-}
-
void oopDesc::release_set_mark(markWord m) {
Atomic::release_store(&_mark, m);
}
+void oopDesc::release_set_mark(HeapWord* mem, markWord m) {
+ Atomic::release_store((markWord*)(((char*)mem) + mark_offset_in_bytes()), m);
+}
+
markWord oopDesc::cas_set_mark(markWord new_mark, markWord old_mark) {
return Atomic::cmpxchg(&_mark, old_mark, new_mark);
}
@@ -82,36 +83,99 @@ markWord oopDesc::cas_set_mark(markWord new_mark, markWord old_mark, atomic_memo
return Atomic::cmpxchg(&_mark, old_mark, new_mark, order);
}
+markWord oopDesc::resolve_mark() const {
+ assert(LockingMode != LM_LEGACY, "Not safe with legacy stack-locking");
+ markWord m = mark();
+ if (m.has_displaced_mark_helper()) {
+ m = m.displaced_mark_helper();
+ }
+ return m;
+}
+
+markWord oopDesc::prototype_mark() const {
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ return klass()->prototype_header();
+ } else {
+#endif
+ return markWord::prototype();
+#ifdef AARCH64
+ }
+#endif
+}
+
void oopDesc::init_mark() {
- set_mark(markWord::prototype());
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ set_mark(klass()->prototype_header());
+ } else
+#endif
+ {
+ set_mark(markWord::prototype());
+ }
}
Klass* oopDesc::klass() const {
+#ifdef _LP64
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ markWord m = resolve_mark();
+ return m.klass();
+ } else
+#endif // AARCH64
if (UseCompressedClassPointers) {
return CompressedKlassPointers::decode_not_null(_metadata._compressed_klass);
- } else {
+ } else
+#endif // _LP64
+ {
return _metadata._klass;
}
}
Klass* oopDesc::klass_or_null() const {
+#ifdef _LP64
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ markWord m = resolve_mark();
+ return m.klass_or_null();
+ } else
+#endif // AARCH64
if (UseCompressedClassPointers) {
return CompressedKlassPointers::decode(_metadata._compressed_klass);
- } else {
+ } else
+#endif // _LP64
+ {
return _metadata._klass;
}
}
Klass* oopDesc::klass_or_null_acquire() const {
+#ifdef _LP64
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ markWord m = mark_acquire();
+ if (m.has_displaced_mark_helper()) {
+ m = m.displaced_mark_helper();
+ }
+ return m.klass_or_null();
+ } else
+#endif // AARCH64
if (UseCompressedClassPointers) {
- narrowKlass nklass = Atomic::load_acquire(&_metadata._compressed_klass);
- return CompressedKlassPointers::decode(nklass);
- } else {
+ narrowKlass nklass = Atomic::load_acquire(&_metadata._compressed_klass);
+ return CompressedKlassPointers::decode(nklass);
+ } else
+#endif // _LP64
+ {
return Atomic::load_acquire(&_metadata._klass);
}
}
Klass* oopDesc::klass_raw() const {
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ return klass();
+ } else
+#endif
if (UseCompressedClassPointers) {
return CompressedKlassPointers::decode_raw(_metadata._compressed_klass);
} else {
@@ -121,6 +185,9 @@ Klass* oopDesc::klass_raw() const {
void oopDesc::set_klass(Klass* k) {
assert(Universe::is_bootstrapping() || (k != nullptr && k->is_klass()), "incorrect Klass");
+#ifdef AARCH64
+ assert(!UseCompactObjectHeaders, "don't set Klass* with compact headers");
+#endif
if (UseCompressedClassPointers) {
_metadata._compressed_klass = CompressedKlassPointers::encode_not_null(k);
} else {
@@ -130,6 +197,9 @@ void oopDesc::set_klass(Klass* k) {
void oopDesc::release_set_klass(HeapWord* mem, Klass* k) {
assert(Universe::is_bootstrapping() || (k != nullptr && k->is_klass()), "incorrect Klass");
+#ifdef AARCH64
+ assert(!UseCompactObjectHeaders, "don't set Klass* with compact headers");
+#endif
char* raw_mem = ((char*)mem + klass_offset_in_bytes());
if (UseCompressedClassPointers) {
Atomic::release_store((narrowKlass*)raw_mem,
@@ -140,6 +210,9 @@ void oopDesc::release_set_klass(HeapWord* mem, Klass* k) {
}
void oopDesc::set_klass_gap(HeapWord* mem, int v) {
+#ifdef AARCH64
+ assert(!UseCompactObjectHeaders, "don't set Klass* gap with compact headers");
+#endif
if (UseCompressedClassPointers) {
*(int*)(((char*)mem) + klass_gap_offset_in_bytes()) = v;
}
@@ -202,6 +275,56 @@ size_t oopDesc::size_given_klass(Klass* klass) {
return s;
}
+#ifdef AARCH64
+Klass* oopDesc::forward_safe_klass_impl(markWord m) const {
+ assert(UseCompactObjectHeaders, "Only get here with compact headers");
+ if (m.is_marked()) {
+ oop fwd = forwardee(m);
+ markWord m2 = fwd->mark();
+ assert(!m2.is_marked() || m2.self_forwarded(), "no double forwarding: this: " PTR_FORMAT " (" INTPTR_FORMAT "), fwd: " PTR_FORMAT " (" INTPTR_FORMAT ")", p2i(this), m.value(), p2i(fwd), m2.value());
+ m = m2;
+ }
+ return m.actual_mark().klass();
+}
+#endif
+
+Klass* oopDesc::forward_safe_klass(markWord m) const {
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ return forward_safe_klass_impl(m);
+ } else
+#endif
+ {
+ return klass();
+ }
+}
+
+Klass* oopDesc::forward_safe_klass() const {
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ return forward_safe_klass_impl(mark());
+ } else
+#endif
+ {
+ return klass();
+ }
+}
+
+size_t oopDesc::forward_safe_size() {
+ return size_given_klass(forward_safe_klass());
+}
+
+void oopDesc::forward_safe_init_mark() {
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ set_mark(forward_safe_klass()->prototype_header());
+ } else
+#endif
+ {
+ set_mark(markWord::prototype());
+ }
+}
+
bool oopDesc::is_instance() const { return klass()->is_instance_klass(); }
bool oopDesc::is_instanceRef() const { return klass()->is_reference_instance_klass(); }
bool oopDesc::is_stackChunk() const { return klass()->is_stack_chunk_instance_klass(); }
@@ -272,19 +395,79 @@ bool oopDesc::is_forwarded() const {
// Used by scavengers
void oopDesc::forward_to(oop p) {
+ assert(p != cast_to_oop(this) || !UseAltGCForwarding, "Must not be called with self-forwarding");
markWord m = markWord::encode_pointer_as_mark(p);
- assert(m.decode_pointer() == p, "encoding must be reversible");
+ assert(forwardee(m) == p, "encoding must be reversible");
set_mark(m);
}
+void oopDesc::forward_to_self() {
+#ifdef _LP64
+ if (UseAltGCForwarding) {
+ markWord m = mark();
+ // If mark is displaced, we need to preserve the real header during GC.
+ // It will be restored to the displaced header after GC.
+ assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint");
+ if (m.has_displaced_mark_helper()) {
+ m = m.displaced_mark_helper();
+ }
+ m = m.set_self_forwarded();
+ assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible");
+ set_mark(m);
+ } else
+#endif
+ {
+ forward_to(oop(this));
+ }
+}
+
oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) {
+ assert(p != cast_to_oop(this) || !UseAltGCForwarding, "Must not be called with self-forwarding");
markWord m = markWord::encode_pointer_as_mark(p);
assert(m.decode_pointer() == p, "encoding must be reversible");
markWord old_mark = cas_set_mark(m, compare, order);
if (old_mark == compare) {
return nullptr;
} else {
- return cast_to_oop(old_mark.decode_pointer());
+ return forwardee(old_mark);
+ }
+}
+
+oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order) {
+#ifdef _LP64
+ if (UseAltGCForwarding) {
+ markWord m = compare;
+ // If mark is displaced, we need to preserve the real header during GC.
+ // It will be restored to the displaced header after GC.
+ assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint");
+ if (m.has_displaced_mark_helper()) {
+ m = m.displaced_mark_helper();
+ }
+ m = m.set_self_forwarded();
+ assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible");
+ markWord old_mark = cas_set_mark(m, compare, order);
+ if (old_mark == compare) {
+ return nullptr;
+ } else {
+ assert(old_mark.is_marked(), "must be marked here");
+ return forwardee(old_mark);
+ }
+ } else
+#endif
+ {
+ return forward_to_atomic(cast_to_oop(this), compare, order);
+ }
+}
+
+oop oopDesc::forwardee(markWord header) const {
+ assert(header.is_marked(), "only decode when actually forwarded");
+#ifdef _LP64
+ if (header.self_forwarded()) {
+ return cast_to_oop(this);
+ } else
+#endif
+ {
+ return cast_to_oop(header.decode_pointer());
}
}
@@ -292,8 +475,7 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde
// The forwardee is used when copying during scavenge and mark-sweep.
// It does need to clear the low two locking- and GC-related bits.
oop oopDesc::forwardee() const {
- assert(is_forwarded(), "only decode when actually forwarded");
- return cast_to_oop(mark().decode_pointer());
+ return forwardee(mark());
}
// The following method needs to be MT safe.
@@ -348,7 +530,8 @@ void oopDesc::oop_iterate_backwards(OopClosureType* cl) {
template <typename OopClosureType>
void oopDesc::oop_iterate_backwards(OopClosureType* cl, Klass* k) {
- assert(k == klass(), "wrong klass");
+ // In this assert, we cannot safely access the Klass* with compact headers.
+ assert(AARCH64_ONLY(UseCompactObjectHeaders ||) k == klass(), "wrong klass");
OopIteratorClosureDispatch::oop_oop_iterate_backwards(cl, this, k);
}
@@ -228,7 +228,8 @@ Klass* TypeArrayKlass::array_klass_or_null() {
}
size_t TypeArrayKlass::oop_size(oop obj) const {
- assert(obj->is_typeArray(),"must be a type array");
+ // In this assert, we cannot safely access the Klass* with compact headers.
+ assert(AARCH64_ONLY(UseCompactObjectHeaders ||) obj->is_typeArray(),"must be a type array");
typeArrayOop t = typeArrayOop(obj);
return t->object_size(this);
}
@@ -35,7 +35,8 @@
class OopIterateClosure;
inline void TypeArrayKlass::oop_oop_iterate_impl(oop obj, OopIterateClosure* closure) {
- assert(obj->is_typeArray(),"must be a type array");
+ // In this assert, we cannot safely access the Klass* with compact headers.
+ assert(AARCH64_ONLY(UseCompactObjectHeaders ||) obj->is_typeArray(),"must be a type array");
// Performance tweak: We skip processing the klass pointer since all
// TypeArrayKlasses are guaranteed processed via the null class loader.
}
@@ -110,7 +110,17 @@ public:
int max_size() const;
void emit(C2_MacroAssembler& masm);
};
-#endif
+
+class C2LoadNKlassStub : public C2CodeStub {
+private:
+ Register _dst;
+public:
+ C2LoadNKlassStub(Register dst) : C2CodeStub(), _dst(dst) {}
+ Register dst() { return _dst; }
+ int max_size() const;
+ void emit(C2_MacroAssembler& masm);
+};
+#endif // _LP64
//-----------------------------C2GeneralStub-----------------------------------
// A generalized stub that can be used to implement an arbitrary stub in a
@@ -1575,8 +1575,17 @@ void AllocateNode::compute_MemBar_redundancy(ciMethod* initializer)
}
Node *AllocateNode::make_ideal_mark(PhaseGVN *phase, Node* obj, Node* control, Node* mem) {
Node* mark_node = nullptr;
- // For now only enable fast locking for non-array types
- mark_node = phase->MakeConX(markWord::prototype().value());
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ Node* klass_node = in(AllocateNode::KlassNode);
+ Node* proto_adr = phase->transform(new AddPNode(klass_node, klass_node, phase->MakeConX(in_bytes(Klass::prototype_header_offset()))));
+ mark_node = LoadNode::make(*phase, control, mem, proto_adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered);
+ } else
+#endif
+ {
+ // For now only enable fast locking for non-array types
+ mark_node = phase->MakeConX(markWord::prototype().value());
+ }
return mark_node;
}
@@ -1696,6 +1696,12 @@ Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_cr
}
}
if (flat->isa_klassptr()) {
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ if (flat->offset() == in_bytes(Klass::prototype_header_offset()))
+ alias_type(idx)->set_rewritable(false);
+ }
+#endif
if (flat->offset() == in_bytes(Klass::super_check_offset_offset()))
alias_type(idx)->set_rewritable(false);
if (flat->offset() == in_bytes(Klass::modifier_flags_offset()))
@@ -4543,8 +4543,8 @@ bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) {
// We depend on hash_mask being at most 32 bits and avoid the use of
// hash_mask_in_place because it could be larger than 32 bits in a 64-bit
// vm: see markWord.hpp.
- Node *hash_mask = _gvn.intcon(markWord::hash_mask);
- Node *hash_shift = _gvn.intcon(markWord::hash_shift);
+ Node *hash_mask = _gvn.intcon(AARCH64_ONLY(UseCompactObjectHeaders ? markWord::hash_mask_compact :) markWord::hash_mask);
+ Node *hash_shift = _gvn.intcon(AARCH64_ONLY(UseCompactObjectHeaders ? markWord::hash_shift_compact :) markWord::hash_shift);
Node *hshifted_header= _gvn.transform(new URShiftXNode(header, hash_shift));
// This hack lets the hash bits live anywhere in the mark object now, as long
// as the shift drops the relevant bits into the low 32 bits. Note that
@@ -1661,7 +1661,9 @@ PhaseMacroExpand::initialize_object(AllocateNode* alloc,
}
rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, TypeX_X->basic_type());
- rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
+ if (AARCH64_ONLY(!UseCompactObjectHeaders) NOT_AARCH64(true)) {
+ rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_METADATA);
+ }
int header_size = alloc->minimum_header_size(); // conservatively small
// Array length
@@ -1885,6 +1885,15 @@ Node *LoadNode::Ideal(PhaseGVN *phase, bool can_reshape) {
const Type*
LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
ciKlass* klass) const {
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
+ // The field is Klass::_prototype_header. Return its (constant) value.
+ assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
+ return TypeX::make(klass->prototype_header());
+ }
+ }
+#endif
if (tkls->offset() == in_bytes(Klass::modifier_flags_offset())) {
// The field is Klass::_modifier_flags. Return its (constant) value.
// (Folds up the 2nd indirection in aClassConstant.getModifiers().)
@@ -2057,6 +2066,15 @@ const Type* LoadNode::Value(PhaseGVN* phase) const {
assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
return TypeInt::make(klass->super_check_offset());
}
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
+ // The field is Klass::_prototype_header. Return its (constant) value.
+ assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
+ return TypeX::make(klass->prototype_header());
+ }
+ }
+#endif
// Compute index into primary_supers array
juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
// Check for overflowing; use unsigned compare to handle the negative case.
@@ -2147,7 +2165,7 @@ const Type* LoadNode::Value(PhaseGVN* phase) const {
}
Node* alloc = is_new_object_mark_load(phase);
- if (alloc != nullptr) {
+ if (AARCH64_ONLY(!UseCompactObjectHeaders &&) alloc != nullptr) {
return TypeX::make(markWord::prototype().value());
}
@@ -320,14 +320,17 @@ JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len
// Zero array here if the caller is deoptimized.
const size_t size = TypeArrayKlass::cast(array_type)->oop_size(result);
BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
- const size_t hs = arrayOopDesc::header_size(elem_type);
- // Align to next 8 bytes to avoid trashing arrays's length.
- const size_t aligned_hs = align_object_offset(hs);
+ size_t hs_bytes = arrayOopDesc::base_offset_in_bytes(elem_type);
+ assert(is_aligned(hs_bytes, BytesPerInt), "must be 4 byte aligned");
HeapWord* obj = cast_from_oop<HeapWord*>(result);
- if (aligned_hs > hs) {
- Copy::zero_to_words(obj+hs, aligned_hs-hs);
+ if (!is_aligned(hs_bytes, BytesPerLong)) {
+ *reinterpret_cast<jint*>(reinterpret_cast<char*>(obj) + hs_bytes) = 0;
+ hs_bytes += BytesPerInt;
}
+
// Optimized zeroing.
+ assert(is_aligned(hs_bytes, BytesPerLong), "must be 8-byte aligned");
+ const size_t aligned_hs = hs_bytes / BytesPerLong;
Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);
}
@@ -5181,12 +5181,12 @@ void TypeAryPtr::dump2( Dict &d, uint depth, outputStream *st ) const {
}
if( _offset != 0 ) {
- int header_size = objArrayOopDesc::header_size() * wordSize;
+ BasicType basic_elem_type = elem()->basic_type();
+ int header_size = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
if( _offset == OffsetTop ) st->print("+undefined");
else if( _offset == OffsetBot ) st->print("+any");
else if( _offset < header_size ) st->print("+%d", _offset);
else {
- BasicType basic_elem_type = elem()->basic_type();
if (basic_elem_type == T_ILLEGAL) {
st->print("+any");
} else {
@@ -68,7 +68,7 @@
#define MAX_OBJECT_SIZE \
- ( arrayOopDesc::header_size(T_DOUBLE) * HeapWordSize \
+ ( arrayOopDesc::base_offset_in_bytes(T_DOUBLE) \
+ ((julong)max_jint * sizeof(double)) )
@@ -82,6 +82,7 @@
#include "runtime/javaCalls.hpp"
#include "runtime/javaThread.inline.hpp"
#include "runtime/jniHandles.inline.hpp"
+#include "runtime/lockStack.hpp"
#include "runtime/os.hpp"
#include "runtime/stackFrameStream.inline.hpp"
#include "runtime/synchronizer.hpp"
@@ -1856,6 +1857,14 @@ WB_ENTRY(jboolean, WB_IsUbsanEnabled(JNIEnv* env))
return (jboolean) WhiteBox::is_ubsan_enabled();
WB_END
+WB_ENTRY(jint, WB_getLockStackCapacity(JNIEnv* env))
+ return (jint) LockStack::CAPACITY;
+WB_END
+
+WB_ENTRY(jboolean, WB_supportsRecursiveLightweightLocking(JNIEnv* env))
+ return (jboolean) VM_Version::supports_recursive_lightweight_locking();
+WB_END
+
WB_ENTRY(jboolean, WB_DeflateIdleMonitors(JNIEnv* env, jobject wb))
log_info(monitorinflation)("WhiteBox initiated DeflateIdleMonitors");
return ObjectSynchronizer::request_deflate_idle_monitors_from_wb();
@@ -2784,6 +2793,8 @@ static JNINativeMethod methods[] = {
{CC"isMonitorInflated0", CC"(Ljava/lang/Object;)Z", (void*)&WB_IsMonitorInflated },
{CC"isAsanEnabled", CC"()Z", (void*)&WB_IsAsanEnabled },
{CC"isUbsanEnabled", CC"()Z", (void*)&WB_IsUbsanEnabled },
+ {CC"getLockStackCapacity", CC"()I", (void*)&WB_getLockStackCapacity },
+ {CC"supportsRecursiveLightweightLocking", CC"()Z", (void*)&WB_supportsRecursiveLightweightLocking },
{CC"forceSafepoint", CC"()V", (void*)&WB_ForceSafepoint },
{CC"forceClassLoaderStatsSafepoint", CC"()V", (void*)&WB_ForceClassLoaderStatsSafepoint },
{CC"getConstantPool0", CC"(Ljava/lang/Class;)J", (void*)&WB_GetConstantPool },
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -183,6 +183,9 @@ class Abstract_VM_Version: AllStatic {
// Does platform support secondary supers table lookup?
constexpr static bool supports_secondary_supers_table() { return false; }
+ // Is recursive lightweight locking implemented for this platform?
+ constexpr static bool supports_recursive_lightweight_locking() { return false; }
+
// Does platform support float16 instructions?
static bool supports_float16() { return false; }
@@ -3110,6 +3110,28 @@ jint Arguments::finalize_vm_init_args(bool patch_mod_javabase) {
UNSUPPORTED_OPTION(ShowRegistersOnAssert);
#endif // CAN_SHOW_REGISTERS_ON_ASSERT
+#ifdef AARCH64
+ if (UseCompactObjectHeaders && UseZGC && !ZGenerational) {
+ if (FLAG_IS_CMDLINE(UseCompactObjectHeaders)) {
+ warning("Single-generational ZGC does not work with compact object headers, disabling UseCompactObjectHeaders");
+ }
+ FLAG_SET_DEFAULT(UseCompactObjectHeaders, false);
+ }
+ if (UseCompactObjectHeaders && FLAG_IS_CMDLINE(UseCompressedClassPointers) && !UseCompressedClassPointers) {
+ warning("Compact object headers require compressed class pointers. Disabling compact object headers.");
+ FLAG_SET_DEFAULT(UseCompactObjectHeaders, false);
+ }
+ if (UseCompactObjectHeaders && LockingMode == LM_LEGACY) {
+ FLAG_SET_DEFAULT(LockingMode, LM_LIGHTWEIGHT);
+ }
+ if (UseCompactObjectHeaders && !UseAltGCForwarding) {
+ FLAG_SET_DEFAULT(UseAltGCForwarding, true);
+ }
+ if (UseCompactObjectHeaders && !UseCompressedClassPointers) {
+ FLAG_SET_DEFAULT(UseCompressedClassPointers, true);
+ }
+#endif
+
return JNI_OK;
}
@@ -3400,13 +3422,22 @@ char* Arguments::get_default_shared_archive_path() {
os::jvm_path(jvm_path, sizeof(jvm_path));
char *end = strrchr(jvm_path, *os::file_separator());
if (end != nullptr) *end = '\0';
- size_t jvm_path_len = strlen(jvm_path);
- size_t file_sep_len = strlen(os::file_separator());
- const size_t len = jvm_path_len + file_sep_len + 20;
- _default_shared_archive_path = NEW_C_HEAP_ARRAY(char, len, mtArguments);
- jio_snprintf(_default_shared_archive_path, len,
- LP64_ONLY(!UseCompressedOops ? "%s%sclasses_nocoops.jsa":) "%s%sclasses.jsa",
- jvm_path, os::file_separator());
+ stringStream tmp;
+ tmp.print("%s%sclasses", jvm_path, os::file_separator());
+#ifdef _LP64
+ if (!UseCompressedOops) {
+ tmp.print_raw("_nocoops");
+ }
+#ifdef AARCH64
+ if (UseCompactObjectHeaders) {
+ // Note that generation of xxx_coh.jsa variants require
+ // --enable-cds-archive-coh at build time
+ tmp.print_raw("_coh");
+ }
+#endif // AARCH64
+#endif // _LP64
+ tmp.print_raw(".jsa");
+ _default_shared_archive_path = os::strdup(tmp.base());
}
return _default_shared_archive_path;
}
@@ -66,19 +66,26 @@ void BasicLock::move_to(oop obj, BasicLock* dest) {
// is small (given the support for inflated fast-path locking in the fast_lock, etc)
// we'll leave that optimization for another time.
- if (displaced_header().is_neutral()) {
- // The object is locked and the resulting ObjectMonitor* will also be
- // locked so it can't be async deflated until ownership is dropped.
- ObjectSynchronizer::inflate_helper(obj);
- // WARNING: We cannot put a check here, because the inflation
- // will not update the displaced header. Once BasicLock is inflated,
- // no one should ever look at its content.
- } else {
- // Typically the displaced header will be 0 (recursive stack lock) or
- // unused_mark. Naively we'd like to assert that the displaced mark
- // value is either 0, neutral, or 3. But with the advent of the
- // store-before-CAS avoidance in fast_lock/compiler_lock_object
- // we can find any flavor mark in the displaced mark.
+ if (LockingMode == LM_LEGACY) {
+ if (displaced_header().is_neutral()) {
+ // The object is locked and the resulting ObjectMonitor* will also be
+ // locked so it can't be async deflated until ownership is dropped.
+ ObjectSynchronizer::inflate_helper(obj);
+ // WARNING: We cannot put a check here, because the inflation
+ // will not update the displaced header. Once BasicLock is inflated,
+ // no one should ever look at its content.
+ } else {
+ // Typically the displaced header will be 0 (recursive stack lock) or
+ // unused_mark. Naively we'd like to assert that the displaced mark
+ // value is either 0, neutral, or 3. But with the advent of the
+ // store-before-CAS avoidance in fast_lock/compiler_lock_object
+ // we can find any flavor mark in the displaced mark.
+ }
+ dest->set_displaced_header(displaced_header());
}
- dest->set_displaced_header(displaced_header());
+#ifdef ASSERT
+ else {
+ dest->set_displaced_header(markWord(badDispHeaderDeopt));
+ }
+#endif
}
@@ -1665,13 +1665,13 @@ bool Deoptimization::relock_objects(JavaThread* thread, GrowableArray<MonitorInf
// We have lost information about the correct state of the lock stack.
// Inflate the locks instead. Enter then inflate to avoid races with
// deflation.
- ObjectSynchronizer::enter(obj, nullptr, deoptee_thread);
+ ObjectSynchronizer::enter_for(obj, nullptr, deoptee_thread);
assert(mon_info->owner()->is_locked(), "object must be locked now");
- ObjectMonitor* mon = ObjectSynchronizer::inflate(deoptee_thread, obj(), ObjectSynchronizer::inflate_cause_vm_internal);
+ ObjectMonitor* mon = ObjectSynchronizer::inflate_for(deoptee_thread, obj(), ObjectSynchronizer::inflate_cause_vm_internal);
assert(mon->owner() == deoptee_thread, "must be");
} else {
BasicLock* lock = mon_info->lock();
- ObjectSynchronizer::enter(obj, lock, deoptee_thread);
+ ObjectSynchronizer::enter_for(obj, lock, deoptee_thread);
assert(mon_info->owner()->is_locked(), "object must be locked now");
}
}
@@ -147,6 +147,7 @@ const size_t minimumSymbolTableSize = 1024;
constraint)
const bool UseCompressedOops = false;
const bool UseCompressedClassPointers = false;
+const bool UseCompactObjectHeaders = false;
const int ObjectAlignmentInBytes = 8;
#endif // _LP64
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2022, Red Hat, Inc. All rights reserved.
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,20 +26,30 @@
#include "precompiled.hpp"
#include "memory/allocation.hpp"
+#include "runtime/globals.hpp"
#include "runtime/lockStack.inline.hpp"
#include "runtime/safepoint.hpp"
#include "runtime/stackWatermark.hpp"
#include "runtime/stackWatermarkSet.inline.hpp"
#include "runtime/thread.hpp"
#include "utilities/copy.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
#include "utilities/ostream.hpp"
+#include <type_traits>
+
const int LockStack::lock_stack_offset = in_bytes(JavaThread::lock_stack_offset());
const int LockStack::lock_stack_top_offset = in_bytes(JavaThread::lock_stack_top_offset());
const int LockStack::lock_stack_base_offset = in_bytes(JavaThread::lock_stack_base_offset());
LockStack::LockStack(JavaThread* jt) :
_top(lock_stack_base_offset), _base() {
+ // Make sure the layout of the object is compatible with the emitted code's assumptions.
+ STATIC_ASSERT(sizeof(_bad_oop_sentinel) == oopSize);
+ STATIC_ASSERT(sizeof(_base[0]) == oopSize);
+ STATIC_ASSERT(std::is_standard_layout<LockStack>::value);
+ STATIC_ASSERT(offsetof(LockStack, _bad_oop_sentinel) == offsetof(LockStack, _base) - oopSize);
#ifdef ASSERT
for (int i = 0; i < CAPACITY; i++) {
_base[i] = nullptr;
@@ -62,11 +73,21 @@ uint32_t LockStack::end_offset() {
void LockStack::verify(const char* msg) const {
assert(LockingMode == LM_LIGHTWEIGHT, "never use lock-stack when light weight locking is disabled");
assert((_top <= end_offset()), "lockstack overflow: _top %d end_offset %d", _top, end_offset());
- assert((_top >= start_offset()), "lockstack underflow: _top %d end_offset %d", _top, start_offset());
+ assert((_top >= start_offset()), "lockstack underflow: _top %d start_offset %d", _top, start_offset());
if (SafepointSynchronize::is_at_safepoint() || (Thread::current()->is_Java_thread() && is_owning_thread())) {
int top = to_index(_top);
for (int i = 0; i < top; i++) {
assert(_base[i] != nullptr, "no zapped before top");
+ if (VM_Version::supports_recursive_lightweight_locking()) {
+ oop o = _base[i];
+ for (; i < top - 1; i++) {
+ // Consecutive entries may be the same
+ if (_base[i + 1] != o) {
+ break;
+ }
+ }
+ }
+
for (int j = i + 1; j < top; j++) {
assert(_base[i] != _base[j], "entries must be unique: %s", msg);
}
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2022, Red Hat, Inc. All rights reserved.
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -35,9 +36,12 @@ class OopClosure;
class outputStream;
class LockStack {
+ friend class LockStackTest;
friend class VMStructs;
-private:
+ JVMCI_ONLY(friend class JVMCIVMStructs;)
+public:
static const int CAPACITY = 8;
+private:
// TODO: It would be very useful if JavaThread::lock_stack_offset() and friends were constexpr,
// but this is currently not the case because we're using offset_of() which is non-constexpr,
@@ -50,6 +54,9 @@ private:
// We do this instead of a simple index into the array because this allows for
// efficient addressing in generated code.
uint32_t _top;
+ // The _bad_oop_sentinel acts as a sentinel value to elide underflow checks in generated code.
+ // The correct layout is statically asserted in the constructor.
+ const uintptr_t _bad_oop_sentinel = badOopVal;
oop _base[CAPACITY];
// Get the owning thread of this lock-stack.
@@ -74,17 +81,35 @@ public:
static uint32_t start_offset();
static uint32_t end_offset();
- // Return true if we have room to push onto this lock-stack, false otherwise.
- inline bool can_push() const;
+ // Returns true if the lock-stack is full. False otherwise.
+ inline bool is_full() const;
// Pushes an oop on this lock-stack.
inline void push(oop o);
- // Pops an oop from this lock-stack.
- inline oop pop();
+ // Get the oldest oop from this lock-stack.
+ // Precondition: This lock-stack must not be empty.
+ inline oop bottom() const;
+
+ // Is the lock-stack empty.
+ inline bool is_empty() const;
+
+ // Check if object is recursive.
+ // Precondition: This lock-stack must contain the oop.
+ inline bool is_recursive(oop o) const;
+
+ // Try recursive enter.
+ // Precondition: This lock-stack must not be full.
+ inline bool try_recursive_enter(oop o);
+
+ // Try recursive exit.
+ // Precondition: This lock-stack must contain the oop.
+ inline bool try_recursive_exit(oop o);
// Removes an oop from an arbitrary location of this lock-stack.
- inline void remove(oop o);
+ // Precondition: This lock-stack must contain the oop.
+ // Returns the number of oops removed.
+ inline size_t remove(oop o);
// Tests whether the oop is on this lock-stack.
inline bool contains(oop o) const;
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2022, Red Hat, Inc. All rights reserved.
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,14 +27,20 @@
#ifndef SHARE_RUNTIME_LOCKSTACK_INLINE_HPP
#define SHARE_RUNTIME_LOCKSTACK_INLINE_HPP
+#include "runtime/lockStack.hpp"
+
#include "memory/iterator.hpp"
#include "runtime/javaThread.hpp"
-#include "runtime/lockStack.hpp"
#include "runtime/safepoint.hpp"
#include "runtime/stackWatermark.hpp"
#include "runtime/stackWatermarkSet.inline.hpp"
+#include "utilities/align.hpp"
+#include "utilities/globalDefinitions.hpp"
inline int LockStack::to_index(uint32_t offset) {
+ assert(is_aligned(offset, oopSize), "Bad alignment: %u", offset);
+ assert((offset <= end_offset()), "lockstack overflow: offset %d end_offset %d", offset, end_offset());
+ assert((offset >= start_offset()), "lockstack underflow: offset %d start_offset %d", offset, start_offset());
return (offset - lock_stack_base_offset) / oopSize;
}
@@ -42,8 +49,8 @@ JavaThread* LockStack::get_thread() const {
return reinterpret_cast<JavaThread*>(addr - lock_stack_offset);
}
-inline bool LockStack::can_push() const {
- return to_index(_top) < CAPACITY;
+inline bool LockStack::is_full() const {
+ return to_index(_top) == CAPACITY;
}
inline bool LockStack::is_owning_thread() const {
@@ -61,45 +68,132 @@ inline void LockStack::push(oop o) {
verify("pre-push");
assert(oopDesc::is_oop(o), "must be");
assert(!contains(o), "entries must be unique");
- assert(can_push(), "must have room");
+ assert(!is_full(), "must have room");
assert(_base[to_index(_top)] == nullptr, "expect zapped entry");
_base[to_index(_top)] = o;
_top += oopSize;
verify("post-push");
}
-inline oop LockStack::pop() {
- verify("pre-pop");
- assert(to_index(_top) > 0, "underflow, probably unbalanced push/pop");
+inline oop LockStack::bottom() const {
+ assert(to_index(_top) > 0, "must contain an oop");
+ return _base[0];
+}
+
+inline bool LockStack::is_empty() const {
+ return to_index(_top) == 0;
+}
+
+inline bool LockStack::is_recursive(oop o) const {
+ if (!VM_Version::supports_recursive_lightweight_locking()) {
+ return false;
+ }
+ verify("pre-is_recursive");
+
+ // This will succeed iff there is a consecutive run of oops on the
+ // lock-stack with a length of at least 2.
+
+ assert(contains(o), "at least one entry must exist");
+ int end = to_index(_top);
+ // Start iterating from the top because the runtime code is more
+ // interested in the balanced locking case when the top oop on the
+ // lock-stack matches o. This will cause the for loop to break out
+ // in the first loop iteration if it is non-recursive.
+ for (int i = end - 1; i > 0; i--) {
+ if (_base[i - 1] == o && _base[i] == o) {
+ verify("post-is_recursive");
+ return true;
+ }
+ if (_base[i] == o) {
+ // o can only occur in one consecutive run on the lock-stack.
+ // Only one of the two oops checked matched o, so this run
+ // must be of length 1 and thus not be recursive. Stop the search.
+ break;
+ }
+ }
+
+ verify("post-is_recursive");
+ return false;
+}
+
+inline bool LockStack::try_recursive_enter(oop o) {
+ if (!VM_Version::supports_recursive_lightweight_locking()) {
+ return false;
+ }
+ verify("pre-try_recursive_enter");
+
+ // This will succeed iff the top oop on the stack matches o.
+ // When successful o will be pushed to the lock-stack creating
+ // a consecutive run at least 2 oops that matches o on top of
+ // the lock-stack.
+
+ assert(!is_full(), "precond");
+
+ int end = to_index(_top);
+ if (end == 0 || _base[end - 1] != o) {
+ // Topmost oop does not match o.
+ verify("post-try_recursive_enter");
+ return false;
+ }
+
+ _base[end] = o;
+ _top += oopSize;
+ verify("post-try_recursive_enter");
+ return true;
+}
+
+inline bool LockStack::try_recursive_exit(oop o) {
+ if (!VM_Version::supports_recursive_lightweight_locking()) {
+ return false;
+ }
+ verify("pre-try_recursive_exit");
+
+ // This will succeed iff the top two oops on the stack matches o.
+ // When successful the top oop will be popped of the lock-stack.
+ // When unsuccessful the lock may still be recursive, in which
+ // case the locking is unbalanced. This case is handled externally.
+
+ assert(contains(o), "entries must exist");
+
+ int end = to_index(_top);
+ if (end <= 1 || _base[end - 1] != o || _base[end - 2] != o) {
+ // The two topmost oops do not match o.
+ verify("post-try_recursive_exit");
+ return false;
+ }
+
_top -= oopSize;
- oop o = _base[to_index(_top)];
-#ifdef ASSERT
- _base[to_index(_top)] = nullptr;
-#endif
- assert(!contains(o), "entries must be unique: " PTR_FORMAT, p2i(o));
- verify("post-pop");
- return o;
+ DEBUG_ONLY(_base[to_index(_top)] = nullptr;)
+ verify("post-try_recursive_exit");
+ return true;
}
-inline void LockStack::remove(oop o) {
+inline size_t LockStack::remove(oop o) {
verify("pre-remove");
assert(contains(o), "entry must be present: " PTR_FORMAT, p2i(o));
+
int end = to_index(_top);
+ int inserted = 0;
for (int i = 0; i < end; i++) {
- if (_base[i] == o) {
- int last = end - 1;
- for (; i < last; i++) {
- _base[i] = _base[i + 1];
+ if (_base[i] != o) {
+ if (inserted != i) {
+ _base[inserted] = _base[i];
}
- _top -= oopSize;
-#ifdef ASSERT
- _base[to_index(_top)] = nullptr;
-#endif
- break;
+ inserted++;
}
}
- assert(!contains(o), "entries must be unique: " PTR_FORMAT, p2i(o));
+
+#ifdef ASSERT
+ for (int i = inserted; i < end; i++) {
+ _base[i] = nullptr;
+ }
+#endif
+
+ uint32_t removed = end - inserted;
+ _top -= removed * oopSize;
+ assert(!contains(o), "entry must have been removed: " PTR_FORMAT, p2i(o));
verify("post-remove");
+ return removed;
}
inline bool LockStack::contains(oop o) const {
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -39,6 +39,7 @@
#include "prims/jvmtiDeferredUpdates.hpp"
#include "prims/jvmtiExport.hpp"
#include "runtime/atomic.hpp"
+#include "runtime/globals.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/javaThread.inline.hpp"
@@ -53,6 +54,7 @@
#include "runtime/sharedRuntime.hpp"
#include "services/threadService.hpp"
#include "utilities/dtrace.hpp"
+#include "utilities/globalDefinitions.hpp"
#include "utilities/globalCounter.inline.hpp"
#include "utilities/macros.hpp"
#include "utilities/preserveException.hpp"
@@ -312,7 +314,70 @@ void ObjectMonitor::ClearSuccOnSuspend::operator()(JavaThread* current) {
// -----------------------------------------------------------------------------
// Enter support
+bool ObjectMonitor::enter_for(JavaThread* locking_thread) {
+ // Used by ObjectSynchronizer::enter_for to enter for another thread.
+ // The monitor is private to or already owned by locking_thread which must be suspended.
+ // So this code may only contend with deflation.
+ assert(locking_thread == Thread::current() || locking_thread->is_obj_deopt_suspend(), "must be");
+
+ // Block out deflation as soon as possible.
+ add_to_contentions(1);
+
+ bool success = false;
+ if (!is_being_async_deflated()) {
+ void* prev_owner = try_set_owner_from(nullptr, locking_thread);
+
+ if (prev_owner == nullptr) {
+ assert(_recursions == 0, "invariant");
+ success = true;
+ } else if (prev_owner == locking_thread) {
+ _recursions++;
+ success = true;
+ } else if (prev_owner == DEFLATER_MARKER) {
+ // Racing with deflation.
+ prev_owner = try_set_owner_from(DEFLATER_MARKER, locking_thread);
+ if (prev_owner == DEFLATER_MARKER) {
+ // Cancelled deflation. Increment contentions as part of the deflation protocol.
+ add_to_contentions(1);
+ success = true;
+ } else if (prev_owner == nullptr) {
+ // At this point we cannot race with deflation as we have both incremented
+ // contentions, seen contention > 0 and seen a DEFLATER_MARKER.
+ // success will only be false if this races with something other than
+ // deflation.
+ prev_owner = try_set_owner_from(nullptr, locking_thread);
+ success = prev_owner == nullptr;
+ }
+ } else if (LockingMode == LM_LEGACY && locking_thread->is_lock_owned((address)prev_owner)) {
+ assert(_recursions == 0, "must be");
+ _recursions = 1;
+ set_owner_from_BasicLock(prev_owner, locking_thread);
+ success = true;
+ }
+ assert(success, "Failed to enter_for: locking_thread=" INTPTR_FORMAT
+ ", this=" INTPTR_FORMAT "{owner=" INTPTR_FORMAT "}, observed owner: " INTPTR_FORMAT,
+ p2i(locking_thread), p2i(this), p2i(owner_raw()), p2i(prev_owner));
+ } else {
+ // Async deflation is in progress and our contentions increment
+ // above lost the race to async deflation. Undo the work and
+ // force the caller to retry.
+ const oop l_object = object();
+ if (l_object != nullptr) {
+ // Attempt to restore the header/dmw to the object's header so that
+ // we only retry once if the deflater thread happens to be slow.
+ install_displaced_markword_in_object(l_object);
+ }
+ }
+
+ add_to_contentions(-1);
+
+ assert(!success || owner_raw() == locking_thread, "must be");
+
+ return success;
+}
+
bool ObjectMonitor::enter(JavaThread* current) {
+ assert(current == JavaThread::current(), "must be");
// The following code is ordered to check the most common cases first
// and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors.
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -217,6 +217,7 @@ private:
static int Knob_SpinLimit;
+ static ByteSize header_offset() { return byte_offset_of(ObjectMonitor, _header); }
static ByteSize owner_offset() { return byte_offset_of(ObjectMonitor, _owner); }
static ByteSize recursions_offset() { return byte_offset_of(ObjectMonitor, _recursions); }
static ByteSize cxq_offset() { return byte_offset_of(ObjectMonitor, _cxq); }
@@ -298,6 +299,7 @@ private:
int contentions() const;
void add_to_contentions(int value);
intx recursions() const { return _recursions; }
+ void set_recursions(size_t recursions);
// JVM/TI GetObjectMonitorUsage() needs this:
ObjectWaiter* first_waiter() { return _WaitSet; }
@@ -332,6 +334,7 @@ private:
void operator()(JavaThread* current);
};
public:
+ bool enter_for(JavaThread* locking_thread);
bool enter(JavaThread* current);
void exit(JavaThread* current, bool not_suspended = true);
void wait(jlong millis, bool interruptible, TRAPS);
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -102,6 +102,12 @@ inline void ObjectMonitor::add_to_contentions(int value) {
Atomic::add(&_contentions, value);
}
+inline void ObjectMonitor::set_recursions(size_t recursions) {
+ assert(_recursions == 0, "must be");
+ assert(has_owner(), "must be owned");
+ _recursions = checked_cast<intx>(recursions);
+}
+
// Clear _owner field; current value must match old_value.
inline void ObjectMonitor::release_clear_owner(void* old_value) {
#ifdef ASSERT
@@ -3257,16 +3257,24 @@ JRT_LEAF(intptr_t*, SharedRuntime::OSR_migration_begin( JavaThread *current) )
kptr2 = fr.next_monitor_in_interpreter_frame(kptr2) ) {
if (kptr2->obj() != nullptr) { // Avoid 'holes' in the monitor array
BasicLock *lock = kptr2->lock();
- // Inflate so the object's header no longer refers to the BasicLock.
- if (lock->displaced_header().is_unlocked()) {
- // The object is locked and the resulting ObjectMonitor* will also be
- // locked so it can't be async deflated until ownership is dropped.
- // See the big comment in basicLock.cpp: BasicLock::move_to().
- ObjectSynchronizer::inflate_helper(kptr2->obj());
+ if (LockingMode == LM_LEGACY) {
+ // Inflate so the object's header no longer refers to the BasicLock.
+ if (lock->displaced_header().is_unlocked()) {
+ // The object is locked and the resulting ObjectMonitor* will also be
+ // locked so it can't be async deflated until ownership is dropped.
+ // See the big comment in basicLock.cpp: BasicLock::move_to().
+ ObjectSynchronizer::inflate_helper(kptr2->obj());
+ }
+ // Now the displaced header is free to move because the
+ // object's header no longer refers to it.
+ buf[i] = (intptr_t)lock->displaced_header().value();
}
- // Now the displaced header is free to move because the
- // object's header no longer refers to it.
- buf[i++] = (intptr_t)lock->displaced_header().value();
+#ifdef ASSERT
+ else {
+ buf[i] = badDispHeaderOSR;
+ }
+#endif
+ i++;
buf[i++] = cast_from_oop<intptr_t>(kptr2->obj());
}
}
@@ -36,6 +36,7 @@
#include "oops/oop.inline.hpp"
#include "runtime/atomic.hpp"
#include "runtime/frame.inline.hpp"
+#include "runtime/globals.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/handshake.hpp"
#include "runtime/interfaceSupport.inline.hpp"
@@ -60,6 +61,7 @@
#include "utilities/align.hpp"
#include "utilities/dtrace.hpp"
#include "utilities/events.hpp"
+#include "utilities/globalDefinitions.hpp"
#include "utilities/globalCounter.inline.hpp"
#include "utilities/linkedlist.hpp"
#include "utilities/preserveException.hpp"
@@ -384,6 +386,19 @@ bool ObjectSynchronizer::quick_enter(oop obj, JavaThread* current,
return false;
}
+ if (LockingMode == LM_LIGHTWEIGHT) {
+ LockStack& lock_stack = current->lock_stack();
+ if (lock_stack.is_full()) {
+ // Always go into runtime if the lock stack is full.
+ return false;
+ }
+ if (lock_stack.try_recursive_enter(obj)) {
+ // Recursive lock successful.
+ current->inc_held_monitor_count();
+ return true;
+ }
+ }
+
const markWord mark = obj->mark();
if (mark.has_monitor()) {
@@ -437,8 +452,9 @@ bool ObjectSynchronizer::quick_enter(oop obj, JavaThread* current,
}
// Handle notifications when synchronizing on value based classes
-void ObjectSynchronizer::handle_sync_on_value_based_class(Handle obj, JavaThread* current) {
- frame last_frame = current->last_frame();
+void ObjectSynchronizer::handle_sync_on_value_based_class(Handle obj, JavaThread* locking_thread) {
+ assert(locking_thread == Thread::current() || locking_thread->is_obj_deopt_suspend(), "must be");
+ frame last_frame = locking_thread->last_frame();
bool bcp_was_adjusted = false;
// Don't decrement bcp if it points to the frame's first instruction. This happens when
// handle_sync_on_value_based_class() is called because of a synchronized method. There
@@ -451,9 +467,9 @@ void ObjectSynchronizer::handle_sync_on_value_based_class(Handle obj, JavaThread
}
if (DiagnoseSyncOnValueBasedClasses == FATAL_EXIT) {
- ResourceMark rm(current);
+ ResourceMark rm;
stringStream ss;
- current->print_active_stack_on(&ss);
+ locking_thread->print_active_stack_on(&ss);
char* base = (char*)strstr(ss.base(), "at");
char* newline = (char*)strchr(ss.base(), '\n');
if (newline != nullptr) {
@@ -462,13 +478,13 @@ void ObjectSynchronizer::handle_sync_on_value_based_class(Handle obj, JavaThread
fatal("Synchronizing on object " INTPTR_FORMAT " of klass %s %s", p2i(obj()), obj->klass()->external_name(), base);
} else {
assert(DiagnoseSyncOnValueBasedClasses == LOG_WARNING, "invalid value for DiagnoseSyncOnValueBasedClasses");
- ResourceMark rm(current);
+ ResourceMark rm;
Log(valuebasedclasses) vblog;
vblog.info("Synchronizing on object " INTPTR_FORMAT " of klass %s", p2i(obj()), obj->klass()->external_name());
- if (current->has_last_Java_frame()) {
+ if (locking_thread->has_last_Java_frame()) {
LogStream info_stream(vblog.info());
- current->print_active_stack_on(&info_stream);
+ locking_thread->print_active_stack_on(&info_stream);
} else {
vblog.info("Cannot find the last Java frame");
}
@@ -495,38 +511,111 @@ static bool useHeavyMonitors() {
// -----------------------------------------------------------------------------
// Monitor Enter/Exit
+
+void ObjectSynchronizer::enter_for(Handle obj, BasicLock* lock, JavaThread* locking_thread) {
+ // When called with locking_thread != Thread::current() some mechanism must synchronize
+ // the locking_thread with respect to the current thread. Currently only used when
+ // deoptimizing and re-locking locks. See Deoptimization::relock_objects
+ assert(locking_thread == Thread::current() || locking_thread->is_obj_deopt_suspend(), "must be");
+ if (!enter_fast_impl(obj, lock, locking_thread)) {
+ // Inflated ObjectMonitor::enter_for is required
+
+ // An async deflation can race after the inflate_for() call and before
+ // enter_for() can make the ObjectMonitor busy. enter_for() returns false
+ // if we have lost the race to async deflation and we simply try again.
+ while (true) {
+ ObjectMonitor* monitor = inflate_for(locking_thread, obj(), inflate_cause_monitor_enter);
+ if (monitor->enter_for(locking_thread)) {
+ return;
+ }
+ assert(monitor->is_being_async_deflated(), "must be");
+ }
+ }
+}
+
+void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, JavaThread* current) {
+ assert(current == Thread::current(), "must be");
+ if (!enter_fast_impl(obj, lock, current)) {
+ // Inflated ObjectMonitor::enter is required
+
+ // An async deflation can race after the inflate() call and before
+ // enter() can make the ObjectMonitor busy. enter() returns false if
+ // we have lost the race to async deflation and we simply try again.
+ while (true) {
+ ObjectMonitor* monitor = inflate(current, obj(), inflate_cause_monitor_enter);
+ if (monitor->enter(current)) {
+ return;
+ }
+ }
+ }
+}
+
// The interpreter and compiler assembly code tries to lock using the fast path
// of this algorithm. Make sure to update that code if the following function is
// changed. The implementation is extremely sensitive to race condition. Be careful.
+bool ObjectSynchronizer::enter_fast_impl(Handle obj, BasicLock* lock, JavaThread* locking_thread) {
-void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, JavaThread* current) {
if (obj->klass()->is_value_based()) {
- handle_sync_on_value_based_class(obj, current);
+ handle_sync_on_value_based_class(obj, locking_thread);
}
- current->inc_held_monitor_count();
+ locking_thread->inc_held_monitor_count();
if (!useHeavyMonitors()) {
if (LockingMode == LM_LIGHTWEIGHT) {
// Fast-locking does not use the 'lock' argument.
- LockStack& lock_stack = current->lock_stack();
- if (lock_stack.can_push()) {
- markWord mark = obj()->mark_acquire();
- while (mark.is_neutral()) {
- // Retry until a lock state change has been observed. cas_set_mark() may collide with non lock bits modifications.
- // Try to swing into 'fast-locked' state.
- assert(!lock_stack.contains(obj()), "thread must not already hold the lock");
- const markWord locked_mark = mark.set_fast_locked();
- const markWord old_mark = obj()->cas_set_mark(locked_mark, mark);
- if (old_mark == mark) {
- // Successfully fast-locked, push object to lock-stack and return.
- lock_stack.push(obj());
- return;
- }
- mark = old_mark;
+ LockStack& lock_stack = locking_thread->lock_stack();
+ if (lock_stack.is_full()) {
+ // We unconditionally make room on the lock stack by inflating
+ // the least recently locked object on the lock stack.
+
+ // About the choice to inflate least recently locked object.
+ // First we must chose to inflate a lock, either some lock on
+ // the lock-stack or the lock that is currently being entered
+ // (which may or may not be on the lock-stack).
+ // Second the best lock to inflate is a lock which is entered
+ // in a control flow where there are only a very few locks being
+ // used, as the costly part of inflated locking is inflation,
+ // not locking. But this property is entirely program dependent.
+ // Third inflating the lock currently being entered on when it
+ // is not present on the lock-stack will result in a still full
+ // lock-stack. This creates a scenario where every deeper nested
+ // monitorenter must call into the runtime.
+ // The rational here is as follows:
+ // Because we cannot (currently) figure out the second, and want
+ // to avoid the third, we inflate a lock on the lock-stack.
+ // The least recently locked lock is chosen as it is the lock
+ // with the longest critical section.
+
+ log_info(monitorinflation)("LockStack capacity exceeded, inflating.");
+ ObjectMonitor* monitor = inflate_for(locking_thread, lock_stack.bottom(), inflate_cause_vm_internal);
+ assert(monitor->owner() == Thread::current(), "must be owner=" PTR_FORMAT " current=" PTR_FORMAT " mark=" PTR_FORMAT,
+ p2i(monitor->owner()), p2i(Thread::current()), monitor->object()->mark_acquire().value());
+ assert(!lock_stack.is_full(), "must have made room here");
+ }
+
+ markWord mark = obj()->mark_acquire();
+ while (mark.is_neutral()) {
+ // Retry until a lock state change has been observed. cas_set_mark() may collide with non lock bits modifications.
+ // Try to swing into 'fast-locked' state.
+ assert(!lock_stack.contains(obj()), "thread must not already hold the lock");
+ const markWord locked_mark = mark.set_fast_locked();
+ const markWord old_mark = obj()->cas_set_mark(locked_mark, mark);
+ if (old_mark == mark) {
+ // Successfully fast-locked, push object to lock-stack and return.
+ lock_stack.push(obj());
+ return true;
}
+ mark = old_mark;
}
- // All other paths fall-through to inflate-enter.
+
+ if (mark.is_fast_locked() && lock_stack.try_recursive_enter(obj())) {
+ // Recursive lock successful.
+ return true;
+ }
+
+ // Failed to fast lock.
+ return false;
} else if (LockingMode == LM_LEGACY) {
markWord mark = obj->mark();
if (mark.is_neutral()) {
@@ -534,15 +623,14 @@ void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, JavaThread* current)
// be visible <= the ST performed by the CAS.
lock->set_displaced_header(mark);
if (mark == obj()->cas_set_mark(markWord::from_pointer(lock), mark)) {
- return;
+ return true;
}
- // Fall through to inflate() ...
} else if (mark.has_locker() &&
- current->is_lock_owned((address) mark.locker())) {
+ locking_thread->is_lock_owned((address) mark.locker())) {
assert(lock != mark.locker(), "must not re-lock the same lock");
assert(lock != (BasicLock*) obj->mark().value(), "don't relock with same BasicLock");
lock->set_displaced_header(markWord::from_pointer(nullptr));
- return;
+ return true;
}
// The object header will never be displaced to this lock,
@@ -550,20 +638,15 @@ void ObjectSynchronizer::enter(Handle obj, BasicLock* lock, JavaThread* current)
// must be non-zero to avoid looking like a re-entrant lock,
// and must not look locked either.
lock->set_displaced_header(markWord::unused_mark());
+
+ // Failed to fast lock.
+ return false;
}
} else if (VerifyHeavyMonitors) {
guarantee((obj->mark().value() & markWord::lock_mask_in_place) != markWord::locked_value, "must not be lightweight/stack-locked");
}
- // An async deflation can race after the inflate() call and before
- // enter() can make the ObjectMonitor busy. enter() returns false if
- // we have lost the race to async deflation and we simply try again.
- while (true) {
- ObjectMonitor* monitor = inflate(current, obj(), inflate_cause_monitor_enter);
- if (monitor->enter(current)) {
- return;
- }
- }
+ return false;
}
void ObjectSynchronizer::exit(oop object, BasicLock* lock, JavaThread* current) {
@@ -573,15 +656,28 @@ void ObjectSynchronizer::exit(oop object, BasicLock* lock, JavaThread* current)
markWord mark = object->mark();
if (LockingMode == LM_LIGHTWEIGHT) {
// Fast-locking does not use the 'lock' argument.
- while (mark.is_fast_locked()) {
- // Retry until a lock state change has been observed. cas_set_mark() may collide with non lock bits modifications.
- const markWord unlocked_mark = mark.set_unlocked();
- const markWord old_mark = object->cas_set_mark(unlocked_mark, mark);
- if (old_mark == mark) {
- current->lock_stack().remove(object);
- return;
+ LockStack& lock_stack = current->lock_stack();
+ if (mark.is_fast_locked() && lock_stack.try_recursive_exit(object)) {
+ // Recursively unlocked.
+ return;
+ }
+
+ if (mark.is_fast_locked() && lock_stack.is_recursive(object)) {
+ // This lock is recursive but is not at the top of the lock stack so we're
+ // doing an unbalanced exit. We have to fall thru to inflation below and
+ // let ObjectMonitor::exit() do the unlock.
+ } else {
+ while (mark.is_fast_locked()) {
+ // Retry until a lock state change has been observed. cas_set_mark() may collide with non lock bits modifications.
+ const markWord unlocked_mark = mark.set_unlocked();
+ const markWord old_mark = object->cas_set_mark(unlocked_mark, mark);
+ if (old_mark == mark) {
+ size_t recursions = lock_stack.remove(object) - 1;
+ assert(recursions == 0, "must not be recursive here");
+ return;
+ }
+ mark = old_mark;
}
- mark = old_mark;
}
} else if (LockingMode == LM_LEGACY) {
markWord dhw = lock->displaced_header();
@@ -631,13 +727,7 @@ void ObjectSynchronizer::exit(oop object, BasicLock* lock, JavaThread* current)
// The ObjectMonitor* can't be async deflated until ownership is
// dropped inside exit() and the ObjectMonitor* must be !is_busy().
ObjectMonitor* monitor = inflate(current, object, inflate_cause_vm_internal);
- if (LockingMode == LM_LIGHTWEIGHT && monitor->is_owner_anonymous()) {
- // It must be owned by us. Pop lock object from lock stack.
- LockStack& lock_stack = current->lock_stack();
- oop popped = lock_stack.pop();
- assert(popped == object, "must be owned by this thread");
- monitor->set_owner_from_anonymous(current);
- }
+ assert(!monitor->is_owner_anonymous(), "must not be");
monitor->exit(current);
}
@@ -905,7 +995,7 @@ static inline intptr_t get_next_hash(Thread* current, oop obj) {
value = v;
}
- value &= markWord::hash_mask;
+ value &= AARCH64_ONLY(UseCompactObjectHeaders ? markWord::hash_mask_compact :) markWord::hash_mask;
if (value == 0) value = 0xBAD;
assert(value != markWord::no_hash, "invariant");
return value;
@@ -1313,15 +1403,28 @@ void ObjectSynchronizer::inflate_helper(oop obj) {
(void)inflate(Thread::current(), obj, inflate_cause_vm_internal);
}
-// Can be called from non JavaThreads (e.g., VMThread) for FastHashCode
-// calculations as part of JVM/TI tagging.
-static bool is_lock_owned(Thread* thread, oop obj) {
- assert(LockingMode == LM_LIGHTWEIGHT, "only call this with new lightweight locking enabled");
- return thread->is_Java_thread() ? JavaThread::cast(thread)->lock_stack().contains(obj) : false;
+ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop obj, const InflateCause cause) {
+ assert(current == Thread::current(), "must be");
+ if (LockingMode == LM_LIGHTWEIGHT && current->is_Java_thread()) {
+ return inflate_impl(JavaThread::cast(current), obj, cause);
+ }
+ return inflate_impl(nullptr, obj, cause);
+}
+
+ObjectMonitor* ObjectSynchronizer::inflate_for(JavaThread* thread, oop obj, const InflateCause cause) {
+ assert(thread == Thread::current() || thread->is_obj_deopt_suspend(), "must be");
+ return inflate_impl(thread, obj, cause);
}
-ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object,
- const InflateCause cause) {
+ObjectMonitor* ObjectSynchronizer::inflate_impl(JavaThread* inflating_thread, oop object, const InflateCause cause) {
+ // The JavaThread* inflating_thread parameter is only used by LM_LIGHTWEIGHT and requires
+ // that the inflating_thread == Thread::current() or is suspended throughout the call by
+ // some other mechanism.
+ // Even with LM_LIGHTWEIGHT the thread might be nullptr when called from a non
+ // JavaThread. (As may still be the case from FastHashCode). However it is only
+ // important for the correctness of the LM_LIGHTWEIGHT algorithm that the thread
+ // is set when called from ObjectSynchronizer::enter from the owning thread,
+ // ObjectSynchronizer::enter_for from any thread, or ObjectSynchronizer::exit.
EventJavaMonitorInflate event;
for (;;) {
@@ -1330,10 +1433,10 @@ ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object,
// The mark can be in one of the following states:
// * inflated - Just return if using stack-locking.
// If using fast-locking and the ObjectMonitor owner
- // is anonymous and the current thread owns the
- // object lock, then we make the current thread the
- // ObjectMonitor owner and remove the lock from the
- // current thread's lock stack.
+ // is anonymous and the inflating_thread owns the
+ // object lock, then we make the inflating_thread
+ // the ObjectMonitor owner and remove the lock from
+ // the inflating_thread's lock stack.
// * fast-locked - Coerce it to inflated from fast-locked.
// * stack-locked - Coerce it to inflated from stack-locked.
// * INFLATING - Busy wait for conversion from stack-locked to
@@ -1345,9 +1448,11 @@ ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object,
ObjectMonitor* inf = mark.monitor();
markWord dmw = inf->header();
assert(dmw.is_neutral(), "invariant: header=" INTPTR_FORMAT, dmw.value());
- if (LockingMode == LM_LIGHTWEIGHT && inf->is_owner_anonymous() && is_lock_owned(current, object)) {
- inf->set_owner_from_anonymous(current);
- JavaThread::cast(current)->lock_stack().remove(object);
+ if (LockingMode == LM_LIGHTWEIGHT && inf->is_owner_anonymous() &&
+ inflating_thread != nullptr && inflating_thread->lock_stack().contains(object)) {
+ inf->set_owner_from_anonymous(inflating_thread);
+ size_t removed = inflating_thread->lock_stack().remove(object);
+ inf->set_recursions(removed - 1);
}
return inf;
}
@@ -1367,12 +1472,12 @@ ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object,
}
// CASE: fast-locked
- // Could be fast-locked either by current or by some other thread.
+ // Could be fast-locked either by the inflating_thread or by some other thread.
//
// Note that we allocate the ObjectMonitor speculatively, _before_
// attempting to set the object's mark to the new ObjectMonitor. If
- // this thread owns the monitor, then we set the ObjectMonitor's
- // owner to this thread. Otherwise, we set the ObjectMonitor's owner
+ // the inflating_thread owns the monitor, then we set the ObjectMonitor's
+ // owner to the inflating_thread. Otherwise, we set the ObjectMonitor's owner
// to anonymous. If we lose the race to set the object's mark to the
// new ObjectMonitor, then we just delete it and loop around again.
//
@@ -1380,10 +1485,10 @@ ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object,
if (LockingMode == LM_LIGHTWEIGHT && mark.is_fast_locked()) {
ObjectMonitor* monitor = new ObjectMonitor(object);
monitor->set_header(mark.set_unlocked());
- bool own = is_lock_owned(current, object);
+ bool own = inflating_thread != nullptr && inflating_thread->lock_stack().contains(object);
if (own) {
- // Owned by us.
- monitor->set_owner_from(nullptr, current);
+ // Owned by inflating_thread.
+ monitor->set_owner_from(nullptr, inflating_thread);
} else {
// Owned by somebody else.
monitor->set_owner_anonymous();
@@ -1393,7 +1498,8 @@ ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object,
if (old_mark == mark) {
// Success! Return inflated monitor.
if (own) {
- JavaThread::cast(current)->lock_stack().remove(object);
+ size_t removed = inflating_thread->lock_stack().remove(object);
+ monitor->set_recursions(removed - 1);
}
// Once the ObjectMonitor is configured and object is associated
// with the ObjectMonitor, it is safe to allow async deflation:
@@ -1403,7 +1509,7 @@ ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object,
// cache lines to avoid false sharing on MP systems ...
OM_PERFDATA_OP(Inflations, inc());
if (log_is_enabled(Trace, monitorinflation)) {
- ResourceMark rm(current);
+ ResourceMark rm;
lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
INTPTR_FORMAT ", type='%s'", p2i(object),
object->mark().value(), object->klass()->external_name());
@@ -1502,7 +1608,7 @@ ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object,
// to avoid false sharing on MP systems ...
OM_PERFDATA_OP(Inflations, inc());
if (log_is_enabled(Trace, monitorinflation)) {
- ResourceMark rm(current);
+ ResourceMark rm;
lsh.print_cr("inflate(has_locker): object=" INTPTR_FORMAT ", mark="
INTPTR_FORMAT ", type='%s'", p2i(object),
object->mark().value(), object->klass()->external_name());
@@ -1546,7 +1652,7 @@ ObjectMonitor* ObjectSynchronizer::inflate(Thread* current, oop object,
// cache lines to avoid false sharing on MP systems ...
OM_PERFDATA_OP(Inflations, inc());
if (log_is_enabled(Trace, monitorinflation)) {
- ResourceMark rm(current);
+ ResourceMark rm;
lsh.print_cr("inflate(neutral): object=" INTPTR_FORMAT ", mark="
INTPTR_FORMAT ", type='%s'", p2i(object),
object->mark().value(), object->klass()->external_name());
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -92,7 +92,18 @@ class ObjectSynchronizer : AllStatic {
// This is the "slow path" version of monitor enter and exit.
static void enter(Handle obj, BasicLock* lock, JavaThread* current);
static void exit(oop obj, BasicLock* lock, JavaThread* current);
+ // Used to enter a monitor for another thread. This requires that the
+ // locking_thread is suspended, and that entering on a potential
+ // inflated monitor may only contend with deflation. That is the obj being
+ // locked on is either already locked by the locking_thread or cannot
+ // escape the locking_thread.
+ static void enter_for(Handle obj, BasicLock* lock, JavaThread* locking_thread);
+private:
+ // Shared implementation for enter and enter_for. Performs all but
+ // inflated monitor enter.
+ static bool enter_fast_impl(Handle obj, BasicLock* lock, JavaThread* locking_thread);
+public:
// Used only to handle jni locks or other unmatched monitor enter/exit
// Internally they will use heavy weight monitor.
static void jni_enter(Handle obj, JavaThread* current);
@@ -113,6 +124,14 @@ class ObjectSynchronizer : AllStatic {
// Inflate light weight monitor to heavy weight monitor
static ObjectMonitor* inflate(Thread* current, oop obj, const InflateCause cause);
+ // Used to inflate a monitor as if it was done from the thread JavaThread.
+ static ObjectMonitor* inflate_for(JavaThread* thread, oop obj, const InflateCause cause);
+
+private:
+ // Shared implementation between the different LockingMode.
+ static ObjectMonitor* inflate_impl(JavaThread* thread, oop obj, const InflateCause cause);
+
+public:
// This version is only for internal use
static void inflate_helper(oop obj);
static const char* inflate_cause_name(const InflateCause cause);
@@ -193,7 +212,7 @@ class ObjectSynchronizer : AllStatic {
static size_t get_gvars_size();
static u_char* get_gvars_stw_random_addr();
- static void handle_sync_on_value_based_class(Handle obj, JavaThread* current);
+ static void handle_sync_on_value_based_class(Handle obj, JavaThread* locking_thread);
};
// ObjectLocker enforces balanced locking and can never throw an
@@ -2591,10 +2591,13 @@
declare_constant(markWord::lock_bits) \
declare_constant(markWord::max_hash_bits) \
declare_constant(markWord::hash_bits) \
+ declare_constant(markWord::hash_bits_compact) \
\
declare_constant(markWord::lock_shift) \
declare_constant(markWord::age_shift) \
declare_constant(markWord::hash_shift) \
+ declare_constant(markWord::hash_shift_compact) \
+ AARCH64_ONLY(declare_constant(markWord::klass_shift)) \
\
declare_constant(markWord::lock_mask) \
declare_constant(markWord::lock_mask_in_place) \
@@ -2602,6 +2605,8 @@
declare_constant(markWord::age_mask_in_place) \
declare_constant(markWord::hash_mask) \
declare_constant(markWord::hash_mask_in_place) \
+ declare_constant(markWord::hash_mask_compact) \
+ declare_constant(markWord::hash_mask_compact_in_place) \
\
declare_constant(markWord::locked_value) \
declare_constant(markWord::unlocked_value) \
new file mode 100644
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_UTILITIES_FASTHASH_HPP
+#define SHARE_UTILITIES_FASTHASH_HPP
+
+#include "memory/allStatic.hpp"
+
+class FastHash : public AllStatic {
+private:
+ static void fullmul64(uint64_t& hi, uint64_t& lo, uint64_t op1, uint64_t op2) {
+#if defined(__SIZEOF_INT128__)
+ __uint128_t prod = static_cast<__uint128_t>(op1) * static_cast<__uint128_t>(op2);
+ hi = static_cast<uint64_t>(prod >> 64);
+ lo = static_cast<uint64_t>(prod >> 0);
+#else
+ /* First calculate all of the cross products. */
+ uint64_t lo_lo = (op1 & 0xFFFFFFFF) * (op2 & 0xFFFFFFFF);
+ uint64_t hi_lo = (op1 >> 32) * (op2 & 0xFFFFFFFF);
+ uint64_t lo_hi = (op1 & 0xFFFFFFFF) * (op2 >> 32);
+ uint64_t hi_hi = (op1 >> 32) * (op2 >> 32);
+
+ /* Now add the products together. These will never overflow. */
+ uint64_t cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
+ uint64_t upper = (hi_lo >> 32) + (cross >> 32) + hi_hi;
+ hi = upper;
+ lo = (cross << 32) | (lo_lo & 0xFFFFFFFF);
+#endif
+ }
+
+ static void fullmul32(uint32_t& hi, uint32_t& lo, uint32_t op1, uint32_t op2) {
+ uint64_t x64 = op1, y64 = op2, xy64 = x64 * y64;
+ hi = (uint32_t)(xy64 >> 32);
+ lo = (uint32_t)(xy64 >> 0);
+ }
+
+ static uint64_t ror(uint64_t x, uint64_t distance) {
+ distance = distance & 0x3F;
+ return (x >> distance) | (x << (64 - distance));
+ }
+
+public:
+ static uint64_t get_hash64(uint64_t x, uint64_t y) {
+ const uint64_t M = 0x8ADAE89C337954D5;
+ const uint64_t A = 0xAAAAAAAAAAAAAAAA; // REPAA
+ const uint64_t H0 = (x ^ y), L0 = (x ^ A);
+
+ uint64_t U0, V0; fullmul64(U0, V0, L0, M);
+ const uint64_t Q0 = (H0 * M);
+ const uint64_t L1 = (Q0 ^ U0);
+
+ uint64_t U1, V1; fullmul64(U1, V1, L1, M);
+ const uint64_t P1 = (V0 ^ M);
+ const uint64_t Q1 = ror(P1, L1);
+ const uint64_t L2 = (Q1 ^ U1);
+ return V1 ^ L2;
+ }
+
+ static uint32_t get_hash32(uint32_t x, uint32_t y) {
+ const uint32_t M = 0x337954D5;
+ const uint32_t A = 0xAAAAAAAA; // REPAA
+ const uint32_t H0 = (x ^ y), L0 = (x ^ A);
+
+ uint32_t U0, V0; fullmul32(U0, V0, L0, M);
+ const uint32_t Q0 = (H0 * M);
+ const uint32_t L1 = (Q0 ^ U0);
+
+ uint32_t U1, V1; fullmul32(U1, V1, L1, M);
+ const uint32_t P1 = (V0 ^ M);
+ const uint32_t Q1 = ror(P1, L1);
+ const uint32_t L2 = (Q1 ^ U1);
+ return V1 ^ L2;
+ }
+};
+
+#endif// SHARE_UTILITIES_FASTHASH_HPP
@@ -1084,7 +1084,8 @@ const juint badHeapWordVal = 0xBAADBABE; // value used to zap
const juint badMetaWordVal = 0xBAADFADE; // value used to zap metadata heap after GC
const int badCodeHeapNewVal= 0xCC; // value used to zap Code heap at allocation
const int badCodeHeapFreeVal = 0xDD; // value used to zap Code heap at deallocation
-
+const intptr_t badDispHeaderDeopt = 0xDE0BD000; // value to fill unused displaced header during deoptimization
+const intptr_t badDispHeaderOSR = 0xDEAD05A0; // value to fill unused displaced header during OSR
// (These must be implemented as #defines because C++ compilers are
// not obligated to inline non-integral constants!)
@@ -24,6 +24,9 @@
package sun.jvm.hotspot.debugger;
+import sun.jvm.hotspot.oops.Mark;
+import sun.jvm.hotspot.runtime.VM;
+
/** <P> DebuggerBase is a recommended base class for debugger
implementations. It can use a PageCache to cache data from the
target process. Note that this class would not be suitable if the
@@ -394,7 +397,15 @@ public abstract class DebuggerBase implements Debugger {
protected long readCompKlassAddressValue(long address)
throws UnmappedAddressException, UnalignedAddressException {
- long value = readCInteger(address, getKlassPtrSize(), true);
+ long value;
+ if (VM.getVM().isCompactObjectHeadersEnabled()) {
+ // With compact headers, the compressed Klass* is currently read from the mark
+ // word. We need to load the whole mark, and shift the upper parts.
+ value = readCInteger(address, machDesc.getAddressSize(), true);
+ value = value >>> Mark.getKlassShift();
+ } else {
+ value = readCInteger(address, getKlassPtrSize(), true);
+ }
if (value != 0) {
value = (long)(narrowKlassBase + (long)(value << narrowKlassShift));
}
@@ -62,4 +62,9 @@ public interface MachineDescription extends Serializable {
default public boolean supports32bitAlignmentOf64bitTypes() {
return false;
}
+
+ /** Indicates whether the underlying machine is Aarch64 platform. */
+ default public boolean isAarch64() {
+ return false;
+ }
}
@@ -36,4 +36,8 @@ public class MachineDescriptionAArch64 extends MachineDescriptionTwosComplement
public boolean isBigEndian() {
return false;
}
+
+ public boolean isAarch64() {
+ return true;
+ }
}
@@ -116,13 +116,6 @@ public class Universe {
heap().printOn(tty);
}
- // Check whether an element of a typeArrayOop with the given type must be
- // aligned 0 mod 8. The typeArrayOop itself must be aligned at least this
- // strongly.
- public static boolean elementTypeShouldBeAligned(BasicType type) {
- return type == BasicType.T_DOUBLE || type == BasicType.T_LONG;
- }
-
// Check whether an object field (static/non-static) of the given type must be
// aligned 0 mod 8.
public static boolean fieldTypeShouldBeAligned(BasicType type) {
@@ -57,32 +57,33 @@ public class Array extends Oop {
private static long lengthOffsetInBytes=0;
private static long typeSize;
+ // Check whether an element of an arrayOop with the given type must be
+ // aligned 0 mod 8. The arrayOop itself must be aligned at least this
+ // strongly.
+ private static boolean elementTypeShouldBeAligned(BasicType type) {
+ if (VM.getVM().isLP64()) {
+ if (type == BasicType.T_OBJECT || type == BasicType.T_ARRAY) {
+ return !VM.getVM().isCompressedOopsEnabled();
+ }
+ }
+ return type == BasicType.T_DOUBLE || type == BasicType.T_LONG;
+ }
+
private static long headerSizeInBytes() {
if (headerSize != 0) {
return headerSize;
}
- if (VM.getVM().isCompressedKlassPointersEnabled()) {
- headerSize = typeSize;
- } else {
- headerSize = VM.getVM().alignUp(typeSize + VM.getVM().getIntSize(),
- VM.getVM().getHeapWordSize());
- }
+ headerSize = lengthOffsetInBytes() + VM.getVM().getIntSize();
return headerSize;
}
- private static long headerSize(BasicType type) {
- if (Universe.elementTypeShouldBeAligned(type)) {
- return alignObjectSize(headerSizeInBytes())/VM.getVM().getHeapWordSize();
- } else {
- return headerSizeInBytes()/VM.getVM().getHeapWordSize();
- }
- }
-
- private long lengthOffsetInBytes() {
+ private static long lengthOffsetInBytes() {
if (lengthOffsetInBytes != 0) {
return lengthOffsetInBytes;
}
- if (VM.getVM().isCompressedKlassPointersEnabled()) {
+ if (VM.getVM().isCompactObjectHeadersEnabled()) {
+ lengthOffsetInBytes = Oop.getHeaderSize();
+ } else if (VM.getVM().isCompressedKlassPointersEnabled()) {
lengthOffsetInBytes = typeSize - VM.getVM().getIntSize();
} else {
lengthOffsetInBytes = typeSize;
@@ -108,7 +109,13 @@ public class Array extends Oop {
}
public static long baseOffsetInBytes(BasicType type) {
- return headerSize(type) * VM.getVM().getHeapWordSize();
+ long typeSizeInBytes = headerSizeInBytes();
+ if (elementTypeShouldBeAligned(type)) {
+ VM vm = VM.getVM();
+ return vm.alignUp(typeSizeInBytes, vm.getVM().getHeapWordSize());
+ } else {
+ return typeSizeInBytes;
+ }
}
public boolean isArray() { return true; }
@@ -55,7 +55,9 @@ public class Instance extends Oop {
// Returns header size in bytes.
public static long getHeaderSize() {
- if (VM.getVM().isCompressedKlassPointersEnabled()) {
+ if (VM.getVM().isCompactObjectHeadersEnabled()) {
+ return Oop.getHeaderSize();
+ } else if (VM.getVM().isCompressedKlassPointersEnabled()) {
return typeSize - VM.getVM().getIntSize();
} else {
return typeSize;
@@ -51,15 +51,22 @@ public class Mark extends VMObject {
lockBits = db.lookupLongConstant("markWord::lock_bits").longValue();
maxHashBits = db.lookupLongConstant("markWord::max_hash_bits").longValue();
hashBits = db.lookupLongConstant("markWord::hash_bits").longValue();
+ hashBitsCompact = db.lookupLongConstant("markWord::hash_bits_compact").longValue();
lockShift = db.lookupLongConstant("markWord::lock_shift").longValue();
ageShift = db.lookupLongConstant("markWord::age_shift").longValue();
hashShift = db.lookupLongConstant("markWord::hash_shift").longValue();
+ hashShiftCompact = db.lookupLongConstant("markWord::hash_shift_compact").longValue();
+ if (VM.getVM().isAarch64()) {
+ klassShift = db.lookupLongConstant("markWord::klass_shift").longValue();
+ }
lockMask = db.lookupLongConstant("markWord::lock_mask").longValue();
lockMaskInPlace = db.lookupLongConstant("markWord::lock_mask_in_place").longValue();
ageMask = db.lookupLongConstant("markWord::age_mask").longValue();
ageMaskInPlace = db.lookupLongConstant("markWord::age_mask_in_place").longValue();
hashMask = db.lookupLongConstant("markWord::hash_mask").longValue();
hashMaskInPlace = db.lookupLongConstant("markWord::hash_mask_in_place").longValue();
+ hashMaskCompact = db.lookupLongConstant("markWord::hash_mask_compact").longValue();
+ hashMaskCompactInPlace = db.lookupLongConstant("markWord::hash_mask_compact_in_place").longValue();
lockedValue = db.lookupLongConstant("markWord::locked_value").longValue();
unlockedValue = db.lookupLongConstant("markWord::unlocked_value").longValue();
monitorValue = db.lookupLongConstant("markWord::monitor_value").longValue();
@@ -78,10 +85,13 @@ public class Mark extends VMObject {
private static long lockBits;
private static long maxHashBits;
private static long hashBits;
+ private static long hashBitsCompact;
private static long lockShift;
private static long ageShift;
private static long hashShift;
+ private static long hashShiftCompact;
+ private static long klassShift;
private static long lockMask;
private static long lockMaskInPlace;
@@ -89,6 +99,8 @@ public class Mark extends VMObject {
private static long ageMaskInPlace;
private static long hashMask;
private static long hashMaskInPlace;
+ private static long hashMaskCompact;
+ private static long hashMaskCompactInPlace;
private static long lockedValue;
private static long unlockedValue;
@@ -102,6 +114,10 @@ public class Mark extends VMObject {
private static long maxAge;
+ public static long getKlassShift() {
+ return klassShift;
+ }
+
public Mark(Address addr) {
super(addr);
}
@@ -174,13 +190,23 @@ public class Mark extends VMObject {
// hash operations
public long hash() {
- return Bits.maskBitsLong(value() >> hashShift, hashMask);
+ if (VM.getVM().isCompactObjectHeadersEnabled()) {
+ return Bits.maskBitsLong(value() >> hashShiftCompact, hashMaskCompact);
+ } else {
+ return Bits.maskBitsLong(value() >> hashShift, hashMask);
+ }
}
public boolean hasNoHash() {
return hash() == noHash;
}
+ public Klass getKlass() {
+ assert(VM.getVM().isCompactObjectHeadersEnabled());
+ assert(!hasMonitor());
+ return (Klass)Metadata.instantiateWrapperFor(addr.getCompKlassAddressAt(0));
+ }
+
// Debugging
public void printOn(PrintStream tty) {
if (isLocked()) {
@@ -46,9 +46,14 @@ public class Oop {
private static synchronized void initialize(TypeDataBase db) throws WrongTypeException {
Type type = db.lookupType("oopDesc");
mark = new CIntField(type.getCIntegerField("_mark"), 0);
- klass = new MetadataField(type.getAddressField("_metadata._klass"), 0);
- compressedKlass = new NarrowKlassField(type.getAddressField("_metadata._compressed_klass"), 0);
- headerSize = type.getSize();
+ if (VM.getVM().isCompactObjectHeadersEnabled()) {
+ Type markType = db.lookupType("markWord");
+ headerSize = markType.getSize();
+ } else {
+ headerSize = type.getSize();
+ klass = new MetadataField(type.getAddressField("_metadata._klass"), 0);
+ compressedKlass = new NarrowKlassField(type.getAddressField("_metadata._compressed_klass"), 0);
+ }
}
private OopHandle handle;
@@ -75,8 +80,21 @@ public class Oop {
// Accessors for declared fields
public Mark getMark() { return new Mark(getHandle()); }
+
+ private static Klass getKlass(Mark mark) {
+ assert(VM.getVM().isCompactObjectHeadersEnabled());
+ if (mark.hasMonitor()) {
+ ObjectMonitor mon = mark.monitor();
+ mark = mon.header();
+ }
+ return mark.getKlass();
+ }
+
public Klass getKlass() {
- if (VM.getVM().isCompressedKlassPointersEnabled()) {
+ if (VM.getVM().isCompactObjectHeadersEnabled()) {
+ assert(VM.getVM().isCompressedKlassPointersEnabled());
+ return getKlass(getMark());
+ } else if (VM.getVM().isCompressedKlassPointersEnabled()) {
return (Klass)compressedKlass.getValue(getHandle());
} else {
return (Klass)klass.getValue(getHandle());
@@ -147,10 +165,12 @@ public class Oop {
void iterateFields(OopVisitor visitor, boolean doVMFields) {
if (doVMFields) {
visitor.doCInt(mark, true);
- if (VM.getVM().isCompressedKlassPointersEnabled()) {
- visitor.doMetadata(compressedKlass, true);
- } else {
- visitor.doMetadata(klass, true);
+ if (!VM.getVM().isCompactObjectHeadersEnabled()) {
+ if (VM.getVM().isCompressedKlassPointersEnabled()) {
+ visitor.doMetadata(compressedKlass, true);
+ } else {
+ visitor.doMetadata(klass, true);
+ }
}
}
}
@@ -206,7 +226,10 @@ public class Oop {
if (handle == null) {
return null;
}
- if (VM.getVM().isCompressedKlassPointersEnabled()) {
+ if (VM.getVM().isCompactObjectHeadersEnabled()) {
+ Mark mark = new Mark(handle);
+ return getKlass(mark);
+ } else if (VM.getVM().isCompressedKlassPointersEnabled()) {
return (Klass)Metadata.instantiateWrapperFor(handle.getCompKlassAddressAt(compressedKlass.getOffset()));
} else {
return (Klass)Metadata.instantiateWrapperFor(handle.getAddressAt(klass.getOffset()));
@@ -96,6 +96,7 @@ public class VM {
private boolean usingServerCompiler;
/** alignment constants */
private boolean isLP64;
+ private boolean isAarch64;
private int bytesPerLong;
private int bytesPerWord;
private int logBytesPerWord;
@@ -148,6 +149,7 @@ public class VM {
private Boolean sharingEnabled;
private Boolean compressedOopsEnabled;
private Boolean compressedKlassPointersEnabled;
+ private Boolean compactObjectHeadersEnabled;
// command line flags supplied to VM - see struct JVMFlag in jvmFlag.hpp
public static final class Flag {
@@ -475,6 +477,7 @@ public class VM {
if (debugger != null) {
isLP64 = debugger.getMachineDescription().isLP64();
+ isAarch64 = debugger.getMachineDescription().isAarch64();
}
bytesPerLong = db.lookupIntConstant("BytesPerLong").intValue();
bytesPerWord = db.lookupIntConstant("BytesPerWord").intValue();
@@ -679,6 +682,15 @@ public class VM {
return isLP64;
}
+ /** Indicates whether the underlying machine supports the Aarch64 data
+ model. This is needed for conditionalizing code in a few places */
+ public boolean isAarch64() {
+ if (Assert.ASSERTS_ENABLED) {
+ Assert.that(isDebugging(), "Debugging system only for now");
+ }
+ return isAarch64;
+ }
+
/** Get bytes-per-long == long/double natural alignment. */
public int getBytesPerLong() {
return bytesPerLong;
@@ -970,6 +982,15 @@ public class VM {
return compressedKlassPointersEnabled.booleanValue();
}
+ public boolean isCompactObjectHeadersEnabled() {
+ if (compactObjectHeadersEnabled == null) {
+ Flag flag = getCommandLineFlag("UseCompactObjectHeaders");
+ compactObjectHeadersEnabled = (flag == null) ? Boolean.FALSE:
+ (flag.getBool()? Boolean.TRUE: Boolean.FALSE);
+ }
+ return compactObjectHeadersEnabled.booleanValue();
+ }
+
public int getObjectAlignmentInBytes() {
if (objectAlignmentInBytes == 0) {
Flag flag = getCommandLineFlag("ObjectAlignmentInBytes");
@@ -26,6 +26,7 @@ package sun.jvm.hotspot.utilities;
import sun.jvm.hotspot.debugger.*;
import sun.jvm.hotspot.oops.Metadata;
+import sun.jvm.hotspot.oops.Oop;
import sun.jvm.hotspot.runtime.*;
import sun.jvm.hotspot.types.*;
@@ -37,26 +38,6 @@ import sun.jvm.hotspot.types.*;
states than the ObjectHeap code. */
public class RobustOopDeterminator {
- private static AddressField klassField;
-
- static {
- VM.registerVMInitializedObserver(new Observer() {
- public void update(Observable o, Object data) {
- initialize(VM.getVM().getTypeDataBase());
- }
- });
- }
-
- private static void initialize(TypeDataBase db) {
- Type type = db.lookupType("oopDesc");
-
- if (VM.getVM().isCompressedKlassPointersEnabled()) {
- klassField = type.getAddressField("_metadata._compressed_klass");
- } else {
- klassField = type.getAddressField("_metadata._klass");
- }
- }
-
public static boolean oopLooksValid(OopHandle oop) {
if (oop == null) {
return false;
@@ -66,11 +47,7 @@ public class RobustOopDeterminator {
}
try {
// Try to instantiate the Klass
- if (VM.getVM().isCompressedKlassPointersEnabled()) {
- Metadata.instantiateWrapperFor(oop.getCompKlassAddressAt(klassField.getOffset()));
- } else {
- Metadata.instantiateWrapperFor(klassField.getValue(oop));
- }
+ Oop.getKlassForOopHandle(oop);
return true;
} catch (AddressException | WrongTypeException e) {
return false;
@@ -22,6 +22,7 @@
*/
#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "oops/oop.inline.hpp"
#include "unittest.hpp"
@@ -55,6 +56,8 @@ TEST_VM(PreservedMarks, iterate_and_restore) {
FakeOop o3;
FakeOop o4;
+ FlagSetting fs(UseAltGCForwarding, false);
+
// Make sure initial marks are correct.
ASSERT_MARK_WORD_EQ(o1.mark(), FakeOop::originalMark());
ASSERT_MARK_WORD_EQ(o2.mark(), FakeOop::originalMark());
new file mode 100644
@@ -0,0 +1,124 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
+#include "oops/markWord.hpp"
+#include "oops/oop.inline.hpp"
+#include "utilities/align.hpp"
+#include "unittest.hpp"
+
+#ifdef _LP64
+#ifndef PRODUCT
+
+static uintptr_t make_mark(uintptr_t target_region, uintptr_t offset) {
+ return (target_region) << 3 | (offset << 4) | 3 /* forwarded */;
+}
+
+static uintptr_t make_fallback() {
+ return ((uintptr_t(1) << 2) /* fallback */ | 3 /* forwarded */);
+}
+
+// Test simple forwarding within the same region.
+TEST_VM(SlidingForwarding, simple) {
+ FlagSetting fs(UseAltGCForwarding, true);
+ HeapWord fakeheap[32] = { nullptr };
+ HeapWord* heap = align_up(fakeheap, 8 * sizeof(HeapWord));
+ oop obj1 = cast_to_oop(&heap[2]);
+ oop obj2 = cast_to_oop(&heap[0]);
+ SlidingForwarding::initialize(MemRegion(&heap[0], &heap[16]), 8);
+ obj1->set_mark(markWord::prototype());
+ SlidingForwarding::begin();
+
+ SlidingForwarding::forward_to<true>(obj1, obj2);
+ ASSERT_EQ(obj1->mark().value(), make_mark(0 /* target_region */, 0 /* offset */));
+ ASSERT_EQ(SlidingForwarding::forwardee<true>(obj1), obj2);
+
+ SlidingForwarding::end();
+}
+
+// Test forwardings crossing 2 regions.
+TEST_VM(SlidingForwarding, tworegions) {
+ FlagSetting fs(UseAltGCForwarding, true);
+ HeapWord fakeheap[32] = { nullptr };
+ HeapWord* heap = align_up(fakeheap, 8 * sizeof(HeapWord));
+ oop obj1 = cast_to_oop(&heap[14]);
+ oop obj2 = cast_to_oop(&heap[2]);
+ oop obj3 = cast_to_oop(&heap[10]);
+ SlidingForwarding::initialize(MemRegion(&heap[0], &heap[16]), 8);
+ obj1->set_mark(markWord::prototype());
+ SlidingForwarding::begin();
+
+ SlidingForwarding::forward_to<true>(obj1, obj2);
+ ASSERT_EQ(obj1->mark().value(), make_mark(0 /* target_region */, 2 /* offset */));
+ ASSERT_EQ(SlidingForwarding::forwardee<true>(obj1), obj2);
+
+ SlidingForwarding::forward_to<true>(obj1, obj3);
+ ASSERT_EQ(obj1->mark().value(), make_mark(1 /* target_region */, 2 /* offset */));
+ ASSERT_EQ(SlidingForwarding::forwardee<true>(obj1), obj3);
+
+ SlidingForwarding::end();
+}
+
+// Test fallback forwardings crossing 4 regions.
+TEST_VM(SlidingForwarding, fallback) {
+ FlagSetting fs(UseAltGCForwarding, true);
+ HeapWord fakeheap[32] = { nullptr };
+ HeapWord* heap = align_up(fakeheap, 8 * sizeof(HeapWord));
+ oop s_obj1 = cast_to_oop(&heap[12]);
+ oop s_obj2 = cast_to_oop(&heap[13]);
+ oop s_obj3 = cast_to_oop(&heap[14]);
+ oop s_obj4 = cast_to_oop(&heap[15]);
+ oop t_obj1 = cast_to_oop(&heap[2]);
+ oop t_obj2 = cast_to_oop(&heap[4]);
+ oop t_obj3 = cast_to_oop(&heap[10]);
+ oop t_obj4 = cast_to_oop(&heap[12]);
+ SlidingForwarding::initialize(MemRegion(&heap[0], &heap[16]), 4);
+ s_obj1->set_mark(markWord::prototype());
+ s_obj2->set_mark(markWord::prototype());
+ s_obj3->set_mark(markWord::prototype());
+ s_obj4->set_mark(markWord::prototype());
+ SlidingForwarding::begin();
+
+ SlidingForwarding::forward_to<true>(s_obj1, t_obj1);
+ ASSERT_EQ(s_obj1->mark().value(), make_mark(0 /* target_region */, 2 /* offset */));
+ ASSERT_EQ(SlidingForwarding::forwardee<true>(s_obj1), t_obj1);
+
+ SlidingForwarding::forward_to<true>(s_obj2, t_obj2);
+ ASSERT_EQ(s_obj2->mark().value(), make_mark(1 /* target_region */, 0 /* offset */));
+ ASSERT_EQ(SlidingForwarding::forwardee<true>(s_obj2), t_obj2);
+
+ SlidingForwarding::forward_to<true>(s_obj3, t_obj3);
+ ASSERT_EQ(s_obj3->mark().value(), make_fallback());
+ ASSERT_EQ(SlidingForwarding::forwardee<true>(s_obj3), t_obj3);
+
+ SlidingForwarding::forward_to<true>(s_obj4, t_obj4);
+ ASSERT_EQ(s_obj4->mark().value(), make_fallback());
+ ASSERT_EQ(SlidingForwarding::forwardee<true>(s_obj4), t_obj4);
+
+ SlidingForwarding::end();
+}
+
+#endif // PRODUCT
+#endif // _LP64
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,19 +27,11 @@
#include "unittest.hpp"
#include "utilities/globalDefinitions.hpp"
-class arrayOopDescTest {
- public:
-
- static int header_size_in_bytes() {
- return arrayOopDesc::header_size_in_bytes();
- }
-};
-
static bool check_max_length_overflow(BasicType type) {
julong length = arrayOopDesc::max_array_length(type);
julong bytes_per_element = type2aelembytes(type);
julong bytes = length * bytes_per_element
- + arrayOopDescTest::header_size_in_bytes();
+ + arrayOopDesc::base_offset_in_bytes(type);
return (julong) (size_t) bytes == bytes;
}
@@ -87,3 +79,58 @@ TEST_VM(arrayOopDesc, narrowOop) {
ASSERT_PRED1(check_max_length_overflow, T_NARROWOOP);
}
// T_VOID and T_ADDRESS are not supported by max_array_length()
+
+TEST_VM(arrayOopDesc, base_offset) {
+#ifdef _LP64
+ if (UseCompactObjectHeaders) {
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_BOOLEAN), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_BYTE), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_SHORT), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_CHAR), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_INT), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_FLOAT), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_LONG), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_DOUBLE), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_OBJECT), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_ARRAY), 12);
+ } else if (UseCompressedClassPointers) {
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_BOOLEAN), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_BYTE), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_SHORT), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_CHAR), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_INT), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_FLOAT), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_LONG), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_DOUBLE), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_OBJECT), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_ARRAY), 16);
+ } else {
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_BOOLEAN), 20);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_BYTE), 20);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_SHORT), 20);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_CHAR), 20);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_INT), 20);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_FLOAT), 20);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_LONG), 24);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_DOUBLE), 24);
+ if (UseCompressedOops) {
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_OBJECT), 20);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_ARRAY), 20);
+ } else {
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_OBJECT), 24);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_ARRAY), 24);
+ }
+ }
+#else
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_BOOLEAN), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_BYTE), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_SHORT), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_CHAR), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_INT), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_FLOAT), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_LONG), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_DOUBLE), 16);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_OBJECT), 12);
+ EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_ARRAY), 12);
+#endif
+}
new file mode 100644
@@ -0,0 +1,69 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "oops/objArrayOop.hpp"
+#include "unittest.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+TEST_VM(objArrayOop, osize) {
+ static const struct {
+ int objal; bool ccp; bool coops; bool coh; int result;
+ } x[] = {
+// ObjAligInB, UseCCP, UseCoops, UseCOH, object size in heap words
+#ifdef _LP64
+ { 8, false, false, false, 4 }, // 20 byte header, 8 byte oops
+ { 8, false, true, false, 3 }, // 20 byte header, 4 byte oops
+ { 8, true, false, false, 3 }, // 16 byte header, 8 byte oops
+ { 8, true, true, false, 3 }, // 16 byte header, 4 byte oops
+ { 16, false, false, false, 4 }, // 20 byte header, 8 byte oops, 16-byte align
+ { 16, false, true, false, 4 }, // 20 byte header, 4 byte oops, 16-byte align
+ { 16, true, false, false, 4 }, // 16 byte header, 8 byte oops, 16-byte align
+ { 16, true, true, false, 4 }, // 16 byte header, 4 byte oops, 16-byte align
+ { 256, false, false, false, 32 }, // 20 byte header, 8 byte oops, 256-byte align
+ { 256, false, true, false, 32 }, // 20 byte header, 4 byte oops, 256-byte align
+ { 256, true, false, false, 32 }, // 16 byte header, 8 byte oops, 256-byte align
+ { 256, true, true, false, 32 }, // 16 byte header, 4 byte oops, 256-byte align
+ { 8, false, false, true, 3 }, // 16 byte header, 8 byte oops
+ { 8, false, true, true, 2 }, // 12 byte header, 4 byte oops
+ { 8, true, false, true, 3 }, // 16 byte header, 8 byte oops
+ { 8, true, true, true, 2 }, // 12 byte header, 4 byte oops
+ { 16, false, false, true, 4 }, // 16 byte header, 8 byte oops, 16-byte align
+ { 16, false, true, true, 2 }, // 12 byte header, 4 byte oops, 16-byte align
+ { 16, true, false, true, 4 }, // 16 byte header, 8 byte oops, 16-byte align
+ { 16, true, true, true, 2 }, // 12 byte header, 4 byte oops, 16-byte align
+ { 256, false, false, true, 32 }, // 16 byte header, 8 byte oops, 256-byte align
+ { 256, false, true, true, 32 }, // 12 byte header, 4 byte oops, 256-byte align
+ { 256, true, false, true, 32 }, // 16 byte header, 8 byte oops, 256-byte align
+ { 256, true, true, true, 32 }, // 12 byte header, 4 byte oops, 256-byte align
+#else
+ { 8, false, false, false, 4 }, // 12 byte header, 4 byte oops, wordsize 4
+#endif
+ { -1, false, false, false, -1 }
+ };
+ for (int i = 0; x[i].result != -1; i++) {
+ if (x[i].objal == (int)ObjectAlignmentInBytes && x[i].ccp == UseCompressedClassPointers && x[i].coops == UseCompressedOops && x[i].coh == UseCompactObjectHeaders) {
+ EXPECT_EQ(objArrayOopDesc::object_size(1), (size_t)x[i].result);
+ }
+ }
+}
@@ -36,7 +36,11 @@ TEST_VM(typeArrayOopDesc, bool_at_put) {
char* addr = align_up(mem, 16);
typeArrayOop o = (typeArrayOop) cast_to_oop(addr);
- o->set_klass(Universe::boolArrayKlassObj());
+ if (UseCompactObjectHeaders) {
+ o->set_mark(Universe::boolArrayKlassObj()->prototype_header());
+ } else {
+ o->set_klass(Universe::boolArrayKlassObj());
+ }
o->set_length(10);
new file mode 100644
@@ -0,0 +1,427 @@
+/*
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
+#include "runtime/lockStack.inline.hpp"
+#include "runtime/os.hpp"
+#include "unittest.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class LockStackTest : public ::testing::Test {
+public:
+ static void push_raw(LockStack& ls, oop obj) {
+ ls._base[ls.to_index(ls._top)] = obj;
+ ls._top += oopSize;
+ }
+
+ static void pop_raw(LockStack& ls) {
+ ls._top -= oopSize;
+#ifdef ASSERT
+ ls._base[ls.to_index(ls._top)] = nullptr;
+#endif
+ }
+
+ static oop at(LockStack& ls, int index) {
+ return ls._base[index];
+ }
+
+ static size_t size(LockStack& ls) {
+ return ls.to_index(ls._top);
+ }
+};
+
+#define recursive_enter(ls, obj) \
+ do { \
+ bool ret = ls.try_recursive_enter(obj); \
+ EXPECT_TRUE(ret); \
+ } while (false)
+
+#define recursive_exit(ls, obj) \
+ do { \
+ bool ret = ls.try_recursive_exit(obj); \
+ EXPECT_TRUE(ret); \
+ } while (false)
+
+TEST_VM_F(LockStackTest, is_recursive) {
+ if (LockingMode != LM_LIGHTWEIGHT || !VM_Version::supports_recursive_lightweight_locking()) {
+ return;
+ }
+
+ JavaThread* THREAD = JavaThread::current();
+ // the thread should be in vm to use locks
+ ThreadInVMfromNative ThreadInVMfromNative(THREAD);
+
+ LockStack& ls = THREAD->lock_stack();
+
+ EXPECT_TRUE(ls.is_empty());
+
+ oop obj0 = Universe::int_mirror();
+ oop obj1 = Universe::float_mirror();
+
+ push_raw(ls, obj0);
+
+ // 0
+ EXPECT_FALSE(ls.is_recursive(obj0));
+
+ push_raw(ls, obj1);
+
+ // 0, 1
+ EXPECT_FALSE(ls.is_recursive(obj0));
+ EXPECT_FALSE(ls.is_recursive(obj1));
+
+ push_raw(ls, obj1);
+
+ // 0, 1, 1
+ EXPECT_FALSE(ls.is_recursive(obj0));
+ EXPECT_TRUE(ls.is_recursive(obj1));
+
+ pop_raw(ls);
+ pop_raw(ls);
+ push_raw(ls, obj0);
+
+ // 0, 0
+ EXPECT_TRUE(ls.is_recursive(obj0));
+
+ push_raw(ls, obj0);
+
+ // 0, 0, 0
+ EXPECT_TRUE(ls.is_recursive(obj0));
+
+ pop_raw(ls);
+ push_raw(ls, obj1);
+
+ // 0, 0, 1
+ EXPECT_TRUE(ls.is_recursive(obj0));
+ EXPECT_FALSE(ls.is_recursive(obj1));
+
+ push_raw(ls, obj1);
+
+ // 0, 0, 1, 1
+ EXPECT_TRUE(ls.is_recursive(obj0));
+ EXPECT_TRUE(ls.is_recursive(obj1));
+
+ // Clear stack
+ pop_raw(ls);
+ pop_raw(ls);
+ pop_raw(ls);
+ pop_raw(ls);
+
+ EXPECT_TRUE(ls.is_empty());
+}
+
+TEST_VM_F(LockStackTest, try_recursive_enter) {
+ if (LockingMode != LM_LIGHTWEIGHT || !VM_Version::supports_recursive_lightweight_locking()) {
+ return;
+ }
+
+ JavaThread* THREAD = JavaThread::current();
+ // the thread should be in vm to use locks
+ ThreadInVMfromNative ThreadInVMfromNative(THREAD);
+
+ LockStack& ls = THREAD->lock_stack();
+
+ EXPECT_TRUE(ls.is_empty());
+
+ oop obj0 = Universe::int_mirror();
+ oop obj1 = Universe::float_mirror();
+
+ ls.push(obj0);
+
+ // 0
+ EXPECT_FALSE(ls.is_recursive(obj0));
+
+ ls.push(obj1);
+
+ // 0, 1
+ EXPECT_FALSE(ls.is_recursive(obj0));
+ EXPECT_FALSE(ls.is_recursive(obj1));
+
+ recursive_enter(ls, obj1);
+
+ // 0, 1, 1
+ EXPECT_FALSE(ls.is_recursive(obj0));
+ EXPECT_TRUE(ls.is_recursive(obj1));
+
+ recursive_exit(ls, obj1);
+ pop_raw(ls);
+ recursive_enter(ls, obj0);
+
+ // 0, 0
+ EXPECT_TRUE(ls.is_recursive(obj0));
+
+ recursive_enter(ls, obj0);
+
+ // 0, 0, 0
+ EXPECT_TRUE(ls.is_recursive(obj0));
+
+ recursive_exit(ls, obj0);
+ push_raw(ls, obj1);
+
+ // 0, 0, 1
+ EXPECT_TRUE(ls.is_recursive(obj0));
+ EXPECT_FALSE(ls.is_recursive(obj1));
+
+ recursive_enter(ls, obj1);
+
+ // 0, 0, 1, 1
+ EXPECT_TRUE(ls.is_recursive(obj0));
+ EXPECT_TRUE(ls.is_recursive(obj1));
+
+ // Clear stack
+ pop_raw(ls);
+ pop_raw(ls);
+ pop_raw(ls);
+ pop_raw(ls);
+
+ EXPECT_TRUE(ls.is_empty());
+}
+
+TEST_VM_F(LockStackTest, contains) {
+ if (LockingMode != LM_LIGHTWEIGHT) {
+ return;
+ }
+
+ const bool test_recursive = VM_Version::supports_recursive_lightweight_locking();
+
+ JavaThread* THREAD = JavaThread::current();
+ // the thread should be in vm to use locks
+ ThreadInVMfromNative ThreadInVMfromNative(THREAD);
+
+ LockStack& ls = THREAD->lock_stack();
+
+ EXPECT_TRUE(ls.is_empty());
+
+ oop obj0 = Universe::int_mirror();
+ oop obj1 = Universe::float_mirror();
+
+ EXPECT_FALSE(ls.contains(obj0));
+
+ ls.push(obj0);
+
+ // 0
+ EXPECT_TRUE(ls.contains(obj0));
+ EXPECT_FALSE(ls.contains(obj1));
+
+ if (test_recursive) {
+ push_raw(ls, obj0);
+
+ // 0, 0
+ EXPECT_TRUE(ls.contains(obj0));
+ EXPECT_FALSE(ls.contains(obj1));
+ }
+
+ push_raw(ls, obj1);
+
+ // 0, 0, 1
+ EXPECT_TRUE(ls.contains(obj0));
+ EXPECT_TRUE(ls.contains(obj1));
+
+ if (test_recursive) {
+ push_raw(ls, obj1);
+
+ // 0, 0, 1, 1
+ EXPECT_TRUE(ls.contains(obj0));
+ EXPECT_TRUE(ls.contains(obj1));
+ }
+
+ pop_raw(ls);
+ if (test_recursive) {
+ pop_raw(ls);
+ pop_raw(ls);
+ }
+ push_raw(ls, obj1);
+
+ // 0, 1
+ EXPECT_TRUE(ls.contains(obj0));
+ EXPECT_TRUE(ls.contains(obj1));
+
+ // Clear stack
+ pop_raw(ls);
+ pop_raw(ls);
+
+ EXPECT_TRUE(ls.is_empty());
+}
+
+TEST_VM_F(LockStackTest, remove) {
+ if (LockingMode != LM_LIGHTWEIGHT) {
+ return;
+ }
+
+ const bool test_recursive = VM_Version::supports_recursive_lightweight_locking();
+
+ JavaThread* THREAD = JavaThread::current();
+ // the thread should be in vm to use locks
+ ThreadInVMfromNative ThreadInVMfromNative(THREAD);
+
+ LockStack& ls = THREAD->lock_stack();
+
+ EXPECT_TRUE(ls.is_empty());
+
+ oop obj0 = Universe::int_mirror();
+ oop obj1 = Universe::float_mirror();
+ oop obj2 = Universe::short_mirror();
+ oop obj3 = Universe::long_mirror();
+
+ push_raw(ls, obj0);
+
+ // 0
+ {
+ size_t removed = ls.remove(obj0);
+ EXPECT_EQ(removed, 1u);
+ EXPECT_FALSE(ls.contains(obj0));
+ }
+
+ if (test_recursive) {
+ push_raw(ls, obj0);
+ push_raw(ls, obj0);
+
+ // 0, 0
+ {
+ size_t removed = ls.remove(obj0);
+ EXPECT_EQ(removed, 2u);
+ EXPECT_FALSE(ls.contains(obj0));
+ }
+ }
+
+ push_raw(ls, obj0);
+ push_raw(ls, obj1);
+
+ // 0, 1
+ {
+ size_t removed = ls.remove(obj0);
+ EXPECT_EQ(removed, 1u);
+ EXPECT_FALSE(ls.contains(obj0));
+ EXPECT_TRUE(ls.contains(obj1));
+
+ ls.remove(obj1);
+ EXPECT_TRUE(ls.is_empty());
+ }
+
+ push_raw(ls, obj0);
+ push_raw(ls, obj1);
+
+ // 0, 1
+ {
+ size_t removed = ls.remove(obj1);
+ EXPECT_EQ(removed, 1u);
+ EXPECT_FALSE(ls.contains(obj1));
+ EXPECT_TRUE(ls.contains(obj0));
+
+ ls.remove(obj0);
+ EXPECT_TRUE(ls.is_empty());
+ }
+
+ if (test_recursive) {
+ push_raw(ls, obj0);
+ push_raw(ls, obj0);
+ push_raw(ls, obj1);
+
+ // 0, 0, 1
+ {
+ size_t removed = ls.remove(obj0);
+ EXPECT_EQ(removed, 2u);
+ EXPECT_FALSE(ls.contains(obj0));
+ EXPECT_TRUE(ls.contains(obj1));
+
+ ls.remove(obj1);
+ EXPECT_TRUE(ls.is_empty());
+ }
+
+ push_raw(ls, obj0);
+ push_raw(ls, obj1);
+ push_raw(ls, obj1);
+
+ // 0, 1, 1
+ {
+ size_t removed = ls.remove(obj1);
+ EXPECT_EQ(removed, 2u);
+ EXPECT_FALSE(ls.contains(obj1));
+ EXPECT_TRUE(ls.contains(obj0));
+
+ ls.remove(obj0);
+ EXPECT_TRUE(ls.is_empty());
+ }
+
+ push_raw(ls, obj0);
+ push_raw(ls, obj1);
+ push_raw(ls, obj1);
+ push_raw(ls, obj2);
+ push_raw(ls, obj2);
+ push_raw(ls, obj2);
+ push_raw(ls, obj2);
+ push_raw(ls, obj3);
+
+ // 0, 1, 1, 2, 2, 2, 2, 3
+ {
+ EXPECT_EQ(size(ls), 8u);
+
+ size_t removed = ls.remove(obj1);
+ EXPECT_EQ(removed, 2u);
+
+ EXPECT_TRUE(ls.contains(obj0));
+ EXPECT_FALSE(ls.contains(obj1));
+ EXPECT_TRUE(ls.contains(obj2));
+ EXPECT_TRUE(ls.contains(obj3));
+
+ EXPECT_EQ(at(ls, 0), obj0);
+ EXPECT_EQ(at(ls, 1), obj2);
+ EXPECT_EQ(at(ls, 2), obj2);
+ EXPECT_EQ(at(ls, 3), obj2);
+ EXPECT_EQ(at(ls, 4), obj2);
+ EXPECT_EQ(at(ls, 5), obj3);
+ EXPECT_EQ(size(ls), 6u);
+
+ removed = ls.remove(obj2);
+ EXPECT_EQ(removed, 4u);
+
+ EXPECT_TRUE(ls.contains(obj0));
+ EXPECT_FALSE(ls.contains(obj1));
+ EXPECT_FALSE(ls.contains(obj2));
+ EXPECT_TRUE(ls.contains(obj3));
+
+ EXPECT_EQ(at(ls, 0), obj0);
+ EXPECT_EQ(at(ls, 1), obj3);
+ EXPECT_EQ(size(ls), 2u);
+
+ removed = ls.remove(obj0);
+ EXPECT_EQ(removed, 1u);
+
+ EXPECT_FALSE(ls.contains(obj0));
+ EXPECT_FALSE(ls.contains(obj1));
+ EXPECT_FALSE(ls.contains(obj2));
+ EXPECT_TRUE(ls.contains(obj3));
+
+ EXPECT_EQ(at(ls, 0), obj3);
+ EXPECT_EQ(size(ls), 1u);
+
+ removed = ls.remove(obj3);
+ EXPECT_EQ(removed, 1u);
+
+ EXPECT_TRUE(ls.is_empty());
+ EXPECT_EQ(size(ls), 0u);
+ }
+ }
+
+ EXPECT_TRUE(ls.is_empty());
+}
@@ -149,6 +149,7 @@ serviceability_ttf_virtual = \
tier1_common = \
sanity/BasicVMTest.java \
gtest/GTestWrapper.java \
+ gtest/LockStackGtests.java \
gtest/MetaspaceGtests.java \
gtest/LargePageGtests.java \
gtest/NMTGtests.java \
@@ -51,7 +51,8 @@ public class TestVectorizationNotRun {
static long[] longArray = new long[size];
@Test
- @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" })
+ @IR(counts = { IRNode.LOAD_VECTOR_L, ">=1", IRNode.STORE_VECTOR, ">=1" },
+ applyIf = {"UseCompactObjectHeaders", "false"})
public static void test(byte[] dest, long[] src) {
for (int i = 0; i < src.length; i++) {
if ((i < 0) || (8 > sizeBytes - i)) {
@@ -141,7 +141,8 @@ public class TestFramework {
"UseZbb",
"UseRVV",
"Xlog",
- "LogCompilation"
+ "LogCompilation",
+ "UseCompactObjectHeaders"
)
);
@@ -270,7 +270,8 @@
@Test
@IR(counts = {IRNode.ADD_VI, "> 0", IRNode.MUL_VI, "> 0", IRNode.ADD_VF, "> 0"},
applyIfPlatform = {"64-bit", "true"},
- applyIfCPUFeatureOr = {"sse4.1", "true", "asimd", "true"})
+ applyIfCPUFeatureOr = {"sse4.1", "true", "asimd", "true"},
+ applyIf = {"UseCompactObjectHeaders", "false"})
static void test6(int[] dataIa, int[] dataIb, float[] dataFa, float[] dataFb,
long[] dataLa, long[] dataLb) {
for (int i = 0; i < RANGE; i+=2) {
@@ -177,6 +177,18 @@
* TestAllocHumongousFragment
*/
+ /*
+ * @test id=g1-alt-forwarding
+ * @summary Make sure G1 can recover from humongous allocation fragmentation, with alt GC forwarding
+ * @key randomness
+ * @requires vm.gc.G1
+ * @library /test/lib
+ *
+ * @run main/othervm -Xlog:gc+region=trace -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx1g -Xms1g
+ * -XX:VerifyGCType=full -XX:+VerifyDuringGC -XX:+VerifyAfterGC -XX:+UseAltGCForwarding
+ * TestAllocHumongousFragment
+ */
+
import java.util.*;
import jdk.test.lib.Utils;
@@ -32,7 +32,7 @@
* @modules java.management
* @build jdk.test.whitebox.WhiteBox
* @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
- * @run main/timeout=240 gc.g1.plab.TestPLABPromotion
+ * @run main/othervm/timeout=240 -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI gc.g1.plab.TestPLABPromotion
*/
package gc.g1.plab;
@@ -45,14 +45,18 @@ import gc.g1.plab.lib.LogParser;
import gc.g1.plab.lib.PLABUtils;
import gc.g1.plab.lib.PlabInfo;
+import jdk.test.lib.Platform;
import jdk.test.lib.process.OutputAnalyzer;
import jdk.test.lib.process.ProcessTools;
+import jdk.test.whitebox.WhiteBox;
/**
* Test checks PLAB promotion of different size objects.
*/
public class TestPLABPromotion {
+ private static final boolean COMPACT_HEADERS = Platform.is64bit() && WhiteBox.getWhiteBox().getBooleanVMFlag("UseCompactObjectHeaders");
+
// GC ID with survivor PLAB statistics
private final static long GC_ID_SURVIVOR_STATS = 1l;
// GC ID with old PLAB statistics
@@ -72,7 +76,7 @@ public class TestPLABPromotion {
private static final int PLAB_SIZE_HIGH = 65536;
private static final int OBJECT_SIZE_SMALL = 10;
private static final int OBJECT_SIZE_MEDIUM = 100;
- private static final int OBJECT_SIZE_HIGH = 3250;
+ private static final int OBJECT_SIZE_HIGH = COMPACT_HEADERS ? 3266 : 3250;
private static final int GC_NUM_SMALL = 1;
private static final int GC_NUM_MEDIUM = 3;
private static final int GC_NUM_HIGH = 7;
@@ -25,7 +25,7 @@
package gc.stress.systemgc;
/*
- * @test TestSystemGCWithG1
+ * @test id=default
* @key stress
* @bug 8190703
* @library /
@@ -33,6 +33,17 @@ package gc.stress.systemgc;
* @summary Stress the G1 GC full GC by allocating objects of different lifetimes concurrently with System.gc().
* @run main/othervm/timeout=300 -Xlog:gc*=info -Xmx512m -XX:+UseG1GC gc.stress.systemgc.TestSystemGCWithG1 270
*/
+
+/*
+ * @test id=alt-forwarding
+ * @key stress
+ * @bug 8190703
+ * @library /
+ * @requires vm.gc.G1
+ * @requires (vm.bits == "64")
+ * @summary Stress the G1 GC full GC by allocating objects of different lifetimes concurrently with System.gc().
+ * @run main/othervm/timeout=300 -XX:+UnlockExperimentalVMOptions -XX:+UseAltGCForwarding -Xlog:gc*=info -Xmx512m -XX:+UseG1GC gc.stress.systemgc.TestSystemGCWithG1 270
+ */
public class TestSystemGCWithG1 {
public static void main(String[] args) throws Exception {
TestSystemGC.main(args);
@@ -25,7 +25,7 @@
package gc.stress.systemgc;
/*
- * @test TestSystemGCWithSerial
+ * @test id=default
* @key stress
* @bug 8190703
* @library /
@@ -33,6 +33,37 @@ package gc.stress.systemgc;
* @summary Stress the Serial GC full GC by allocating objects of different lifetimes concurrently with System.gc().
* @run main/othervm/timeout=300 -Xlog:gc*=info -Xmx512m -XX:+UseSerialGC gc.stress.systemgc.TestSystemGCWithSerial 270
*/
+
+/*
+ * @test id=alt-forwarding
+ * @key stress
+ * @bug 8190703
+ * @library /
+ * @requires vm.gc.Serial
+ * @summary Stress the Serial GC full GC by allocating objects of different lifetimes concurrently with System.gc().
+ * @run main/othervm/timeout=300 -XX:+UnlockExperimentalVMOptions -XX:+UseAltGCForwarding -Xlog:gc*=info -Xmx512m -XX:+UseSerialGC gc.stress.systemgc.TestSystemGCWithSerial 270
+ */
+
+/*
+ * @test id=alt-forwarding-unaligned
+ * @key stress
+ * @bug 8190703
+ * @library /
+ * @requires vm.gc.Serial
+ * @summary Stress the Serial GC full GC by allocating objects of different lifetimes concurrently with System.gc().
+ * @run main/othervm/timeout=300 -XX:+UnlockExperimentalVMOptions -XX:+UseAltGCForwarding -Xlog:gc*=info -Xmx700m -XX:+UseSerialGC gc.stress.systemgc.TestSystemGCWithSerial 270
+ */
+
+/*
+ * @test id=alt-forwarding-large-heap
+ * @key stress
+ * @bug 8190703
+ * @library /
+ * @requires vm.gc.Serial
+ * @requires (vm.bits == "64") & (os.maxMemory >= 6G)
+ * @summary Stress the Serial GC full GC by allocating objects of different lifetimes concurrently with System.gc().
+ * @run main/othervm/timeout=300 -XX:+UnlockExperimentalVMOptions -XX:+UseAltGCForwarding -Xlog:gc*=info -Xmx6g -XX:+UseSerialGC gc.stress.systemgc.TestSystemGCWithSerial 270
+ */
public class TestSystemGCWithSerial {
public static void main(String[] args) throws Exception {
TestSystemGC.main(args);
@@ -40,6 +40,22 @@ package gc.stress.systemgc;
* -XX:+UseShenandoahGC
* gc.stress.systemgc.TestSystemGCWithShenandoah 270
*/
+/*
+ * @test id=alt-forwarding
+ * @key stress
+ * @library /
+ * @requires vm.gc.Shenandoah
+ * @summary Stress the Shenandoah GC full GC by allocating objects of different lifetimes concurrently with System.gc().
+ *
+ * @run main/othervm/timeout=300 -Xlog:gc*=info -Xmx512m -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions
+ * -XX:+UseShenandoahGC -XX:+UseAltGCForwarding
+ * -XX:+ShenandoahVerify
+ * gc.stress.systemgc.TestSystemGCWithShenandoah 270
+ *
+ * @run main/othervm/timeout=300 -Xlog:gc*=info -Xmx512m -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions
+ * -XX:+UseShenandoahGC
+ * gc.stress.systemgc.TestSystemGCWithShenandoah 270
+ */
/*
* @test id=iu
new file mode 100644
@@ -0,0 +1,56 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * This tests object array sizes by running gtests with different settings.
+ */
+
+/* @test id=with-coops-with-ccp
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=arrayOop -XX:+UseCompressedClassPointers -XX:+UseCompressedOops
+ */
+/* @test id=with-coops-no-ccp
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=arrayOop -XX:-UseCompressedClassPointers -XX:+UseCompressedOops
+ */
+/* @test id=no-coops-with-ccp
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=arrayOop -XX:+UseCompressedClassPointers -XX:-UseCompressedOops
+ */
+/* @test id=no-coops-no-ccp
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=arrayOop -XX:-UseCompressedClassPointers -XX:-UseCompressedOops
+ */
new file mode 100644
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/* @test
+ * @summary Run LockStack gtests with LockingMode=2
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @requires vm.flagless
+ * @run main/native GTestWrapper --gtest_filter=LockStackTest* -XX:+UnlockExperimentalVMOptions -XX:LockingMode=2
+ */
new file mode 100644
@@ -0,0 +1,85 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * This tests object array sizes by running gtests with different settings.
+ */
+
+/* @test id=with-coops-with-ccp
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=objArrayOop -XX:+UseCompressedClassPointers -XX:+UseCompressedOops
+ */
+/* @test id=with-coops-no-ccp
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=objArrayOop -XX:-UseCompressedClassPointers -XX:+UseCompressedOops
+ */
+/* @test id=no-coops-with-ccp
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=objArrayOop -XX:+UseCompressedClassPointers -XX:-UseCompressedOops
+ */
+/* @test id=no-coops-no-ccp
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=objArrayOop -XX:-UseCompressedClassPointers -XX:-UseCompressedOops
+ */
+
+/* @test id=with-coops-with-ccp-large-align
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=objArrayOop -XX:+UseCompressedClassPointers -XX:+UseCompressedOops -XX:ObjAlignmentInBytes=256
+ */
+/* @test id=with-coops-no-ccp-large-align
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=objArrayOop -XX:-UseCompressedClassPointers -XX:+UseCompressedOops -XX:ObjAlignmentInBytes=256
+ */
+/* @test id=no-coops-with-ccp-large-align
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=objArrayOop -XX:+UseCompressedClassPointers -XX:-UseCompressedOops -XX:ObjAlignmentInBytes=256
+ */
+/* @test id=no-coops-no-ccp-large-align
+ * @summary Run object array size tests with compressed oops and compressed class pointers
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * java.xml
+ * @run main/native GTestWrapper --gtest_filter=objArrayOop -XX:-UseCompressedClassPointers -XX:-UseCompressedOops -XX:ObjAlignmentInBytes=256
+ */
new file mode 100644
@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test id=default
+ * @library /test/lib /
+ * @modules java.base/jdk.internal.misc
+ * java.management
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI BaseOffsets
+ */
+/*
+ * @test id=no-coops
+ * @library /test/lib /
+ * @requires vm.bits == "64"
+ * @modules java.base/jdk.internal.misc
+ * java.management
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:-UseCompressedOops BaseOffsets
+ */
+/*
+ * @test id=no-ccp
+ * @library /test/lib /
+ * @requires vm.bits == "64"
+ * @modules java.base/jdk.internal.misc
+ * java.management
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:-UseCompressedClassPointers BaseOffsets
+ */
+/*
+ * @test id=no-compact-headers
+ * @library /test/lib /
+ * @requires vm.bits == "64"
+ * @modules java.base/jdk.internal.misc
+ * java.management
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UnlockExperimentalVMOptions -XX:+UseCompactObjectHeaders BaseOffsets
+ */
+
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Comparator;
+import jdk.internal.misc.Unsafe;
+
+import jdk.test.lib.Asserts;
+import jdk.test.lib.Platform;
+import jdk.test.whitebox.WhiteBox;
+
+public class BaseOffsets {
+
+ static class LIClass {
+ public int i;
+ }
+
+ public static final WhiteBox WB = WhiteBox.getWhiteBox();
+
+ static final long INT_OFFSET;
+ static final int INT_ARRAY_OFFSET;
+ static final int LONG_ARRAY_OFFSET;
+ static {
+ if (!Platform.is64bit()) {
+ INT_OFFSET = 8;
+ INT_ARRAY_OFFSET = 12;
+ LONG_ARRAY_OFFSET = 16;
+ } else if (WB.getBooleanVMFlag("UseCompactObjectHeaders")) {
+ INT_OFFSET = 8;
+ INT_ARRAY_OFFSET = 12;
+ LONG_ARRAY_OFFSET = 16;
+ } else if (WB.getBooleanVMFlag("UseCompressedClassPointers")) {
+ INT_OFFSET = 12;
+ INT_ARRAY_OFFSET = 16;
+ LONG_ARRAY_OFFSET = 16;
+ } else {
+ INT_OFFSET = 16;
+ INT_ARRAY_OFFSET = 20;
+ LONG_ARRAY_OFFSET = 24;
+ }
+ }
+
+ static public void main(String[] args) {
+ Unsafe unsafe = Unsafe.getUnsafe();
+ Class c = LIClass.class;
+ Field[] fields = c.getFields();
+ for (int i = 0; i < fields.length; i++) {
+ long offset = unsafe.objectFieldOffset(fields[i]);
+ if (fields[i].getType() == int.class) {
+ Asserts.assertEquals(offset, INT_OFFSET, "Misplaced int field");
+ } else {
+ Asserts.fail("Unexpected field type");
+ }
+ }
+
+ Asserts.assertEquals(unsafe.arrayBaseOffset(boolean[].class), INT_ARRAY_OFFSET, "Misplaced boolean array base");
+ Asserts.assertEquals(unsafe.arrayBaseOffset(byte[].class), INT_ARRAY_OFFSET, "Misplaced byte array base");
+ Asserts.assertEquals(unsafe.arrayBaseOffset(char[].class), INT_ARRAY_OFFSET, "Misplaced char array base");
+ Asserts.assertEquals(unsafe.arrayBaseOffset(short[].class), INT_ARRAY_OFFSET, "Misplaced short array base");
+ Asserts.assertEquals(unsafe.arrayBaseOffset(int[].class), INT_ARRAY_OFFSET, "Misplaced int array base");
+ Asserts.assertEquals(unsafe.arrayBaseOffset(long[].class), LONG_ARRAY_OFFSET, "Misplaced long array base");
+ Asserts.assertEquals(unsafe.arrayBaseOffset(float[].class), INT_ARRAY_OFFSET, "Misplaced float array base");
+ Asserts.assertEquals(unsafe.arrayBaseOffset(double[].class), LONG_ARRAY_OFFSET, "Misplaced double array base");
+ boolean narrowOops = System.getProperty("java.vm.compressedOopsMode") != null ||
+ !Platform.is64bit();
+ int expected_objary_offset = narrowOops ? INT_ARRAY_OFFSET : LONG_ARRAY_OFFSET;
+ Asserts.assertEquals(unsafe.arrayBaseOffset(Object[].class), expected_objary_offset, "Misplaced object array base");
+ }
+}
@@ -25,20 +25,24 @@
* @test
* @bug 8239014
* @summary -XX:-UseEmptySlotsInSupers sometime fails to reproduce the layout of the old code
- * @library /test/lib
+ * @library /test/lib /
* @modules java.base/jdk.internal.misc
* java.management
* @requires vm.bits == "64" & vm.opt.final.UseCompressedOops == true & vm.gc != "Z"
- * @run main/othervm -XX:+UseCompressedClassPointers -XX:-UseEmptySlotsInSupers OldLayoutCheck
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UseCompressedClassPointers -XX:-UseEmptySlotsInSupers OldLayoutCheck
*/
/*
* @test
* @requires vm.bits == "32"
- * @library /test/lib
+ * @library /test/lib /
* @modules java.base/jdk.internal.misc
* java.management
- * @run main/othervm -XX:-UseEmptySlotsInSupers OldLayoutCheck
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:-UseEmptySlotsInSupers OldLayoutCheck
*/
import java.lang.reflect.Field;
@@ -48,6 +52,7 @@ import jdk.internal.misc.Unsafe;
import jdk.test.lib.Asserts;
import jdk.test.lib.Platform;
+import jdk.test.whitebox.WhiteBox;
public class OldLayoutCheck {
@@ -56,10 +61,21 @@ public class OldLayoutCheck {
public int i;
}
- // 32-bit VMs: @0: 8 byte header, @8: long field, @16: int field
- // 64-bit VMs: @0: 12 byte header, @12: int field, @16: long field
- static final long INT_OFFSET = Platform.is64bit() ? 12L : 16L;
- static final long LONG_OFFSET = Platform.is64bit() ? 16L : 8L;
+ public static final WhiteBox WB = WhiteBox.getWhiteBox();
+
+ // 32-bit VMs/compact headers: @0: 8 byte header, @8: long field, @16: int field
+ // 64-bit VMs: @0: 12 byte header, @12: int field, @16: long field
+ static final long INT_OFFSET;
+ static final long LONG_OFFSET;
+ static {
+ if (!Platform.is64bit() || WB.getBooleanVMFlag("UseCompactObjectHeaders")) {
+ INT_OFFSET = 16L;
+ LONG_OFFSET = 8L;
+ } else {
+ INT_OFFSET = 12L;
+ LONG_OFFSET = 16L;
+ }
+ }
static public void main(String[] args) {
Unsafe unsafe = Unsafe.getUnsafe();
new file mode 100644
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test CdsDifferentCompactObjectHeaders
+ * @summary Testing CDS (class data sharing) using opposite compact object header settings.
+ * Using different compact bject headers setting for each dump/load pair.
+ * This is a negative test; using compact header setting for loading that
+ * is different from compact headers for creating a CDS file
+ * should fail when loading.
+ * @requires vm.cds
+ * @requires vm.bits == 64
+ * @library /test/lib
+ * @run driver CdsDifferentCompactObjectHeaders
+ */
+
+import jdk.test.lib.cds.CDSTestUtils;
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.Platform;
+
+public class CdsDifferentCompactObjectHeaders {
+
+ public static void main(String[] args) throws Exception {
+ createAndLoadSharedArchive(true, false);
+ createAndLoadSharedArchive(false, true);
+ }
+
+ // Parameters are object alignment expressed in bytes
+ private static void
+ createAndLoadSharedArchive(boolean createCompactHeaders, boolean loadCompactHeaders)
+ throws Exception {
+ String createCompactHeadersArg = "-XX:" + (createCompactHeaders ? "+" : "-") + "UseCompactObjectHeaders";
+ String loadCompactHeadersArg = "-XX:" + (loadCompactHeaders ? "+" : "-") + "UseCompactObjectHeaders";
+ String expectedErrorMsg =
+ String.format(
+ "The shared archive file's UseCompactObjectHeaders setting (%s)" +
+ " does not equal the current UseCompactObjectHeaders setting (%s)",
+ createCompactHeaders ? "enabled" : "disabled",
+ loadCompactHeaders ? "enabled" : "disabled");
+
+ CDSTestUtils.createArchiveAndCheck("-XX:+UnlockExperimentalVMOptions", createCompactHeadersArg);
+
+ OutputAnalyzer out = CDSTestUtils.runWithArchive("-Xlog:cds", "-XX:+UnlockExperimentalVMOptions", loadCompactHeadersArg);
+ CDSTestUtils.checkExecExpectError(out, 1, expectedErrorMsg);
+ }
+}
@@ -56,6 +56,7 @@ public class TestZGCWithCDS {
public final static String ERR_MSG = "The saved state of UseCompressedOops and UseCompressedClassPointers is different from runtime, CDS will be disabled.";
public static void main(String... args) throws Exception {
String zGenerational = args[0];
+ String compactHeaders = "-XX:" + (zGenerational.equals("-XX:+ZGenerational") ? "+" : "-") + "UseCompactObjectHeaders";
String helloJar = JarBuilder.build("hello", "Hello");
System.out.println("0. Dump with ZGC");
OutputAnalyzer out = TestCommon
@@ -63,6 +64,8 @@ public class TestZGCWithCDS {
new String[] {"Hello"},
"-XX:+UseZGC",
zGenerational,
+ "-XX:+UnlockExperimentalVMOptions",
+ compactHeaders,
"-Xlog:cds");
out.shouldContain("Dumping shared data to file:");
out.shouldHaveExitValue(0);
@@ -72,6 +75,8 @@ public class TestZGCWithCDS {
.exec(helloJar,
"-XX:+UseZGC",
zGenerational,
+ "-XX:+UnlockExperimentalVMOptions",
+ compactHeaders,
"-Xlog:cds",
"Hello");
out.shouldContain(HELLO);
@@ -83,6 +88,8 @@ public class TestZGCWithCDS {
"-XX:-UseZGC",
"-XX:+UseCompressedOops", // in case turned off by vmoptions
"-XX:+UseCompressedClassPointers", // by jtreg
+ "-XX:+UnlockExperimentalVMOptions",
+ compactHeaders,
"-Xlog:cds",
"Hello");
out.shouldContain(UNABLE_TO_USE_ARCHIVE);
@@ -107,6 +114,8 @@ public class TestZGCWithCDS {
"-XX:+UseSerialGC",
"-XX:-UseCompressedOops",
"-XX:+UseCompressedClassPointers",
+ "-XX:+UnlockExperimentalVMOptions",
+ compactHeaders,
"-Xlog:cds",
"Hello");
out.shouldContain(HELLO);
@@ -130,6 +139,8 @@ public class TestZGCWithCDS {
"-XX:+UseSerialGC",
"-XX:+UseCompressedOops",
"-XX:+UseCompressedClassPointers",
+ "-XX:+UnlockExperimentalVMOptions",
+ compactHeaders,
"-Xlog:cds",
"Hello");
out.shouldContain(UNABLE_TO_USE_ARCHIVE);
@@ -143,6 +154,8 @@ public class TestZGCWithCDS {
"-XX:+UseSerialGC",
"-XX:-UseCompressedOops",
"-XX:+UseCompressedClassPointers",
+ "-XX:+UnlockExperimentalVMOptions",
+ compactHeaders,
"-Xlog:cds");
out.shouldContain("Dumping shared data to file:");
out.shouldHaveExitValue(0);
@@ -152,6 +165,8 @@ public class TestZGCWithCDS {
.exec(helloJar,
"-XX:+UseZGC",
zGenerational,
+ "-XX:+UnlockExperimentalVMOptions",
+ compactHeaders,
"-Xlog:cds",
"Hello");
out.shouldContain(HELLO);
@@ -144,6 +144,7 @@ public class TestAutoCreateSharedArchiveNoDefaultArchive {
private static void removeDefaultArchives(String java_home_dst, String variant) {
removeDefaultArchive(java_home_dst, variant, "");
removeDefaultArchive(java_home_dst, variant, "_nocoops");
+ removeDefaultArchive(java_home_dst, variant, "_coh");
}
private static void removeDefaultArchive(String java_home_dst, String variant, String suffix) {
new file mode 100644
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test TestLockStackCapacity
+ * @summary Tests the interaction between recursive lightweight locking and
+ * when the lock stack capacity is exceeded.
+ * @requires vm.flagless
+ * @library /testlibrary /test/lib
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -Xint -XX:LockingMode=2 TestLockStackCapacity
+ */
+
+import jdk.test.lib.Asserts;
+import jdk.test.whitebox.WhiteBox;
+import jtreg.SkippedException;
+
+public class TestLockStackCapacity {
+ static final WhiteBox WB = WhiteBox.getWhiteBox();
+ static final int LockingMode = WB.getIntVMFlag("LockingMode").intValue();
+ static final int LM_LIGHTWEIGHT = 2;
+
+ static class SynchronizedObject {
+ static final SynchronizedObject OUTER = new SynchronizedObject();
+ static final SynchronizedObject INNER = new SynchronizedObject();
+ static final int LockStackCapacity = WB.getLockStackCapacity();
+
+ synchronized void runInner(int depth) {
+ assertNotInflated();
+ if (depth == 1) {
+ return;
+ } else {
+ runInner(depth - 1);
+ }
+ assertNotInflated();
+ }
+
+ synchronized void runOuter(int depth, SynchronizedObject inner) {
+ assertNotInflated();
+ if (depth == 1) {
+ inner.runInner(LockStackCapacity);
+ } else {
+ runOuter(depth - 1, inner);
+ }
+ assertInflated();
+ }
+
+ public static void runTest() {
+ // Test Requires a capacity of at least 2.
+ Asserts.assertGTE(LockStackCapacity, 2);
+
+ // Just checking
+ OUTER.assertNotInflated();
+ INNER.assertNotInflated();
+
+ synchronized(OUTER) {
+ OUTER.assertNotInflated();
+ INNER.assertNotInflated();
+ OUTER.runOuter(LockStackCapacity - 1, INNER);
+
+ OUTER.assertInflated();
+ INNER.assertNotInflated();
+ }
+ }
+
+ void assertNotInflated() {
+ Asserts.assertFalse(WB.isMonitorInflated(this));
+ }
+
+ void assertInflated() {
+ Asserts.assertTrue(WB.isMonitorInflated(this));
+ }
+ }
+
+ public static void main(String... args) throws Exception {
+ if (LockingMode != LM_LIGHTWEIGHT) {
+ throw new SkippedException("Test only valid for LM_LIGHTWEIGHT");
+ }
+
+ if (!WB.supportsRecursiveLightweightLocking()) {
+ throw new SkippedException("Test only valid if LM_LIGHTWEIGHT supports recursion");
+ }
+
+ SynchronizedObject.runTest();
+ }
+}
@@ -120,7 +120,46 @@
* -XX:-DoEscapeAnalysis -XX:-EliminateAllocations -XX:+EliminateLocks -XX:+EliminateNestedLocks
* -XX:+IgnoreUnrecognizedVMOptions -XX:+DeoptimizeObjectsALot
*
+ * @bug 8324881
+ * @comment Regression test for using the wrong thread when logging during re-locking from deoptimization.
+ *
+ * @comment DiagnoseSyncOnValueBasedClasses=2 will cause logging when locking on \@ValueBased objects.
+ * @run driver EATests
+ * -XX:+UnlockDiagnosticVMOptions
+ * -Xms256m -Xmx256m
+ * -Xbootclasspath/a:.
+ * -XX:CompileCommand=dontinline,*::dontinline_*
+ * -XX:+WhiteBoxAPI
+ * -Xbatch
+ * -XX:+DoEscapeAnalysis -XX:+EliminateAllocations -XX:+EliminateLocks -XX:+EliminateNestedLocks
+ * -XX:+UnlockExperimentalVMOptions -XX:LockingMode=1
+ * -XX:DiagnoseSyncOnValueBasedClasses=2
+ *
+ * @comment Re-lock may inflate monitors when re-locking, which cause monitorinflation trace logging.
+ * @run driver EATests
+ * -XX:+UnlockDiagnosticVMOptions
+ * -Xms256m -Xmx256m
+ * -Xbootclasspath/a:.
+ * -XX:CompileCommand=dontinline,*::dontinline_*
+ * -XX:+WhiteBoxAPI
+ * -Xbatch
+ * -XX:+DoEscapeAnalysis -XX:+EliminateAllocations -XX:+EliminateLocks -XX:+EliminateNestedLocks
+ * -XX:+UnlockExperimentalVMOptions -XX:LockingMode=2
+ * -Xlog:monitorinflation=trace:file=monitorinflation.log
+ *
+ * @comment Re-lock may race with deflation.
+ * @run driver EATests
+ * -XX:+UnlockDiagnosticVMOptions
+ * -Xms256m -Xmx256m
+ * -Xbootclasspath/a:.
+ * -XX:CompileCommand=dontinline,*::dontinline_*
+ * -XX:+WhiteBoxAPI
+ * -Xbatch
+ * -XX:+DoEscapeAnalysis -XX:+EliminateAllocations -XX:+EliminateLocks -XX:+EliminateNestedLocks
+ * -XX:+UnlockExperimentalVMOptions -XX:LockingMode=0
+ * -XX:GuaranteedAsyncDeflationInterval=1000
*/
+
/**
* @test
* @bug 8227745
@@ -254,12 +293,14 @@ class EATestsTarget {
new EARelockingRecursiveTarget() .run();
new EARelockingNestedInflatedTarget() .run();
new EARelockingNestedInflated_02Target() .run();
+ new EARelockingNestedInflated_03Target() .run();
new EARelockingArgEscapeLWLockedInCalleeFrameTarget() .run();
new EARelockingArgEscapeLWLockedInCalleeFrame_2Target() .run();
new EARelockingArgEscapeLWLockedInCalleeFrameNoRecursiveTarget() .run();
new EAGetOwnedMonitorsTarget() .run();
new EAEntryCountTarget() .run();
new EARelockingObjectCurrentlyWaitingOnTarget() .run();
+ new EARelockingValueBasedTarget() .run();
// Test cases that require deoptimization even though neither
// locks nor allocations are eliminated at the point where
@@ -374,12 +415,14 @@ public class EATests extends TestScaffold {
new EARelockingRecursive() .run(this);
new EARelockingNestedInflated() .run(this);
new EARelockingNestedInflated_02() .run(this);
+ new EARelockingNestedInflated_03() .run(this);
new EARelockingArgEscapeLWLockedInCalleeFrame() .run(this);
new EARelockingArgEscapeLWLockedInCalleeFrame_2() .run(this);
new EARelockingArgEscapeLWLockedInCalleeFrameNoRecursive() .run(this);
new EAGetOwnedMonitors() .run(this);
new EAEntryCount() .run(this);
new EARelockingObjectCurrentlyWaitingOn() .run(this);
+ new EARelockingValueBased() .run(this);
// Test cases that require deoptimization even though neither
// locks nor allocations are eliminated at the point where
@@ -2013,6 +2056,94 @@ class EARelockingNestedInflated_02Target extends EATestCaseBaseTarget {
/////////////////////////////////////////////////////////////////////////////
+/**
+ * Like {@link EARelockingNestedInflated_02} with the difference that the
+ * inflation of the lock happens because of contention.
+ */
+class EARelockingNestedInflated_03 extends EATestCaseBaseDebugger {
+
+ public void runTestCase() throws Exception {
+ BreakpointEvent bpe = resumeTo(TARGET_TESTCASE_BASE_NAME, "dontinline_brkpt", "()V");
+ printStack(bpe.thread());
+ @SuppressWarnings("unused")
+ ObjectReference o = getLocalRef(bpe.thread().frame(2), XYVAL_NAME, "l1");
+ }
+}
+
+class EARelockingNestedInflated_03Target extends EATestCaseBaseTarget {
+
+ public XYVal lockInflatedByContention;
+ public boolean doLockNow;
+ public EATestCaseBaseTarget testCase;
+
+ @Override
+ public void setUp() {
+ super.setUp();
+ testMethodDepth = 2;
+ lockInflatedByContention = new XYVal(1, 1);
+ testCase = this;
+ }
+
+ @Override
+ public void warmupDone() {
+ super.warmupDone();
+ // Use new lock. lockInflatedByContention might have been inflated because of recursion.
+ lockInflatedByContention = new XYVal(1, 1);
+ // Start thread that tries to enter lockInflatedByContention while the main thread owns it -> inflation
+ TestScaffold.newThread(() -> {
+ while (true) {
+ synchronized (testCase) {
+ try {
+ if (doLockNow) {
+ doLockNow = false; // reset for main thread
+ testCase.notify();
+ break;
+ }
+ testCase.wait();
+ } catch (InterruptedException e) { /* ignored */ }
+ }
+ }
+ synchronized (lockInflatedByContention) { // will block and trigger inflation
+ msg(Thread.currentThread().getName() + ": acquired lockInflatedByContention");
+ }
+ }, testCaseName + ": Lock Contender (test thread)").start();
+ }
+
+ public void dontinline_testMethod() {
+ @SuppressWarnings("unused")
+ XYVal xy = new XYVal(1, 1); // scalar replaced
+ XYVal l1 = lockInflatedByContention; // read by debugger
+ synchronized (l1) {
+ testMethod_inlined(l1);
+ }
+ }
+
+ public void testMethod_inlined(XYVal l2) {
+ synchronized (l2) { // eliminated nested locking
+ dontinline_notifyOtherThread();
+ dontinline_brkpt();
+ }
+ }
+
+ public void dontinline_notifyOtherThread() {
+ if (!warmupDone) {
+ return;
+ }
+ synchronized (testCase) {
+ doLockNow = true;
+ testCase.notify();
+ // wait for other thread to reset doLockNow again
+ while (doLockNow) {
+ try {
+ testCase.wait();
+ } catch (InterruptedException e) { /* ignored */ }
+ }
+ }
+ }
+}
+
+/////////////////////////////////////////////////////////////////////////////
+
/**
* Checks if an eliminated lock of an ArgEscape object l1 can be relocked if
* l1 is locked in a callee frame.
@@ -2228,6 +2359,32 @@ class EARelockingObjectCurrentlyWaitingOnTarget extends EATestCaseBaseTarget {
}
}
+
+/////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Test relocking eliminated @ValueBased object.
+ */
+class EARelockingValueBased extends EATestCaseBaseDebugger {
+
+ public void runTestCase() throws Exception {
+ BreakpointEvent bpe = resumeTo(TARGET_TESTCASE_BASE_NAME, "dontinline_brkpt", "()V");
+ printStack(bpe.thread());
+ @SuppressWarnings("unused")
+ ObjectReference o = getLocalRef(bpe.thread().frame(1), Integer.class.getName(), "l1");
+ }
+}
+
+class EARelockingValueBasedTarget extends EATestCaseBaseTarget {
+
+ public void dontinline_testMethod() {
+ Integer l1 = new Integer(255);
+ synchronized (l1) {
+ dontinline_brkpt();
+ }
+ }
+}
+
/////////////////////////////////////////////////////////////////////////////
//
// Test cases that require deoptimization even though neither locks
@@ -301,6 +301,7 @@ import jdk.test.whitebox.WhiteBox;
public class GetObjectSizeIntrinsicsTest extends ASimpleInstrumentationTestCase {
+ private static final boolean COMPACT_HEADERS = Platform.is64bit() && WhiteBox.getWhiteBox().getBooleanVMFlag("UseCompactObjectHeaders");
static final Boolean COMPRESSED_OOPS = WhiteBox.getWhiteBox().getBooleanVMFlag("UseCompressedOops");
static final long REF_SIZE = (COMPRESSED_OOPS == null || COMPRESSED_OOPS == true) ? 4 : 8;
@@ -313,6 +314,9 @@ public class GetObjectSizeIntrinsicsTest extends ASimpleInstrumentationTestCase
static final int LARGE_INT_ARRAY_SIZE = 1024*1024*1024 + 1024;
static final int LARGE_OBJ_ARRAY_SIZE = (4096/(int)REF_SIZE)*1024*1024 + 1024;
+ static final boolean CCP = WhiteBox.getWhiteBox().getBooleanVMFlag("UseCompressedClassPointers");
+ static final int ARRAY_HEADER_SIZE = CCP ? 16 : (Platform.is64bit() ? 20 : 16);
+
final String mode;
public GetObjectSizeIntrinsicsTest(String name, String mode) {
@@ -371,15 +375,25 @@ public class GetObjectSizeIntrinsicsTest extends ASimpleInstrumentationTestCase
return (v + a - 1) / a * a;
}
+ private static long expectedSmallObjSize() {
+ long size;
+ if (!Platform.is64bit() || COMPACT_HEADERS) {
+ size = 8;
+ } else {
+ size = 16;
+ }
+ return roundUp(size, OBJ_ALIGN);
+ }
+
private void testSize_newObject() {
- long expected = roundUp(Platform.is64bit() ? 16 : 8, OBJ_ALIGN);
+ long expected = expectedSmallObjSize();
for (int c = 0; c < ITERS; c++) {
assertEquals(expected, fInst.getObjectSize(new Object()));
}
}
private void testSize_localObject() {
- long expected = roundUp(Platform.is64bit() ? 16 : 8, OBJ_ALIGN);
+ long expected = expectedSmallObjSize();
Object o = new Object();
for (int c = 0; c < ITERS; c++) {
assertEquals(expected, fInst.getObjectSize(o));
@@ -389,14 +403,14 @@ public class GetObjectSizeIntrinsicsTest extends ASimpleInstrumentationTestCase
static Object staticO = new Object();
private void testSize_fieldObject() {
- long expected = roundUp(Platform.is64bit() ? 16 : 8, OBJ_ALIGN);
+ long expected = expectedSmallObjSize();
for (int c = 0; c < ITERS; c++) {
assertEquals(expected, fInst.getObjectSize(staticO));
}
}
private void testSize_newSmallIntArray() {
- long expected = roundUp(4L*SMALL_ARRAY_SIZE + 16, OBJ_ALIGN);
+ long expected = roundUp(4L*SMALL_ARRAY_SIZE + ARRAY_HEADER_SIZE, OBJ_ALIGN);
for (int c = 0; c < ITERS; c++) {
assertEquals(expected, fInst.getObjectSize(new int[SMALL_ARRAY_SIZE]));
}
@@ -404,7 +418,7 @@ public class GetObjectSizeIntrinsicsTest extends ASimpleInstrumentationTestCase
private void testSize_localSmallIntArray() {
int[] arr = new int[SMALL_ARRAY_SIZE];
- long expected = roundUp(4L*SMALL_ARRAY_SIZE + 16, OBJ_ALIGN);
+ long expected = roundUp(4L*SMALL_ARRAY_SIZE + ARRAY_HEADER_SIZE, OBJ_ALIGN);
for (int c = 0; c < ITERS; c++) {
assertEquals(expected, fInst.getObjectSize(arr));
}
@@ -413,14 +427,14 @@ public class GetObjectSizeIntrinsicsTest extends ASimpleInstrumentationTestCase
static int[] smallArr = new int[SMALL_ARRAY_SIZE];
private void testSize_fieldSmallIntArray() {
- long expected = roundUp(4L*SMALL_ARRAY_SIZE + 16, OBJ_ALIGN);
+ long expected = roundUp(4L*SMALL_ARRAY_SIZE + ARRAY_HEADER_SIZE, OBJ_ALIGN);
for (int c = 0; c < ITERS; c++) {
assertEquals(expected, fInst.getObjectSize(smallArr));
}
}
private void testSize_newSmallObjArray() {
- long expected = roundUp(REF_SIZE*SMALL_ARRAY_SIZE + 16, OBJ_ALIGN);
+ long expected = roundUp(REF_SIZE*SMALL_ARRAY_SIZE + ARRAY_HEADER_SIZE, OBJ_ALIGN);
for (int c = 0; c < ITERS; c++) {
assertEquals(expected, fInst.getObjectSize(new Object[SMALL_ARRAY_SIZE]));
}
@@ -428,7 +442,7 @@ public class GetObjectSizeIntrinsicsTest extends ASimpleInstrumentationTestCase
private void testSize_localSmallObjArray() {
Object[] arr = new Object[SMALL_ARRAY_SIZE];
- long expected = roundUp(REF_SIZE*SMALL_ARRAY_SIZE + 16, OBJ_ALIGN);
+ long expected = roundUp(REF_SIZE*SMALL_ARRAY_SIZE + ARRAY_HEADER_SIZE, OBJ_ALIGN);
for (int c = 0; c < ITERS; c++) {
assertEquals(expected, fInst.getObjectSize(arr));
}
@@ -437,7 +451,7 @@ public class GetObjectSizeIntrinsicsTest extends ASimpleInstrumentationTestCase
static Object[] smallObjArr = new Object[SMALL_ARRAY_SIZE];
private void testSize_fieldSmallObjArray() {
- long expected = roundUp(REF_SIZE*SMALL_ARRAY_SIZE + 16, OBJ_ALIGN);
+ long expected = roundUp(REF_SIZE*SMALL_ARRAY_SIZE + ARRAY_HEADER_SIZE, OBJ_ALIGN);
for (int c = 0; c < ITERS; c++) {
assertEquals(expected, fInst.getObjectSize(smallObjArr));
}
@@ -445,7 +459,7 @@ public class GetObjectSizeIntrinsicsTest extends ASimpleInstrumentationTestCase
private void testSize_localLargeIntArray() {
int[] arr = new int[LARGE_INT_ARRAY_SIZE];
- long expected = roundUp(4L*LARGE_INT_ARRAY_SIZE + 16, OBJ_ALIGN);
+ long expected = roundUp(4L*LARGE_INT_ARRAY_SIZE + ARRAY_HEADER_SIZE, OBJ_ALIGN);
for (int c = 0; c < ITERS; c++) {
assertEquals(expected, fInst.getObjectSize(arr));
}
@@ -453,7 +467,7 @@ public class GetObjectSizeIntrinsicsTest extends ASimpleInstrumentationTestCase
private void testSize_localLargeObjArray() {
Object[] arr = new Object[LARGE_OBJ_ARRAY_SIZE];
- long expected = roundUp(REF_SIZE*LARGE_OBJ_ARRAY_SIZE + 16, OBJ_ALIGN);
+ long expected = roundUp(REF_SIZE*LARGE_OBJ_ARRAY_SIZE + ARRAY_HEADER_SIZE, OBJ_ALIGN);
for (int c = 0; c < ITERS; c++) {
assertEquals(expected, fInst.getObjectSize(arr));
}
@@ -26,6 +26,7 @@ import java.io.File;
import jdk.test.lib.JDKToolFinder;
import jdk.test.lib.Platform;
import jdk.test.lib.process.*;
+import jdk.test.whitebox.WhiteBox;
import tests.Helper;
@@ -44,7 +45,9 @@ import jtreg.SkippedException;
* jdk.jlink/jdk.tools.jimage
* jdk.compiler
* @build tests.*
- * @run main CDSPluginTest
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -Xbootclasspath/a:. CDSPluginTest
*/
public class CDSPluginTest {
@@ -61,9 +64,12 @@ public class CDSPluginTest {
}
var module = "cds";
+ boolean COMPACT_HEADERS =
+ Platform.isAArch64() && WhiteBox.getWhiteBox().getBooleanVMFlag("UseCompactObjectHeaders");
helper.generateDefaultJModule(module);
- var image = helper.generateDefaultImage(new String[] { "--generate-cds-archive" },
- module)
+ String[] options = COMPACT_HEADERS ? new String[] { "--generate-cds-archive", "--add-options", "-XX:+UnlockExperimentalVMOptions -XX:+UseCompactObjectHeaders" }
+ : new String[] { "--generate-cds-archive" };
+ var image = helper.generateDefaultImage(options, module)
.assertSuccess();
String subDir;
@@ -75,12 +81,15 @@ public class CDSPluginTest {
}
subDir += "server" + sep;
- if (Platform.isAArch64() || Platform.isX64()) {
+
+ String suffix = COMPACT_HEADERS ? "_coh.jsa" : ".jsa";
+
+ if (Platform.isAArch64()) {
helper.checkImage(image, module, null, null,
- new String[] { subDir + "classes.jsa", subDir + "classes_nocoops.jsa" });
+ new String[] { subDir + "classes" + suffix, subDir + "classes_nocoops" + suffix });
} else {
helper.checkImage(image, module, null, null,
- new String[] { subDir + "classes.jsa" });
+ new String[] { subDir + "classes" + suffix });
}
}
}
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -119,6 +119,10 @@ public class WhiteBox {
return isMonitorInflated0(obj);
}
+ public native int getLockStackCapacity();
+
+ public native boolean supportsRecursiveLightweightLocking();
+
public native void forceSafepoint();
public native void forceClassLoaderStatsSafepoint();
--
2.50.1
