* Copyright (c) 2022 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ecmascript/mem/barriers_get-inl.h"
#include "ecmascript/mem/local_cmc/cc_evacuator-inl.h"
#include "ecmascript/mem/work_manager-inl.h"
#include "common_components/heap/allocator/region_desc.h"
#include "common_components/mutator/mutator.h"
#include "ecmascript/runtime.h"
namespace panda::ecmascript {
void Barriers::Update(const JSThread *thread, uintptr_t slotAddr, Region *objectRegion, TaggedObject *value,
Region *valueRegion, WriteBarrierType writeType)
{
ASSERT(!g_isEnableCMCGC);
if (valueRegion->InSharedHeap()) {
return;
}
auto heap = thread->GetEcmaVM()->GetHeap();
TaggedObject *heapValue = JSTaggedValue(value).GetHeapObject();
if (heap->IsConcurrentFullMark()) {
if (valueRegion->InCollectSet() && !objectRegion->InYoungSpaceOrCSet()) {
objectRegion->AtomicInsertCrossRegionRSet(slotAddr);
}
} else if (heap->IsYoungMark()) {
#if USE_STICKY_CMS_GC
if (heapValue->IsInOld()) {
return;
}
#else
if (!valueRegion->InYoungSpace()) {
return;
}
#endif
}
if (valueRegion->IsFreshRegion()) {
if (valueRegion->NonAtomicMark(heapValue)) {
valueRegion->IncreaseAliveObject(heapValue->GetSize());
}
} else if (writeType == WriteBarrierType::NORMAL && valueRegion->AtomicMark(heapValue)) {
std::atomic_thread_fence(std::memory_order_seq_cst);
heap->GetWorkManager()->GetWorkNodeHolder(MAIN_THREAD_INDEX)->Push(heapValue);
}
}
void Barriers::UpdateShared(const JSThread *thread, uintptr_t slotAddr, Region *objectRegion, TaggedObject *value,
Region *valueRegion)
{
ASSERT(!g_isEnableCMCGC);
ASSERT(DaemonThread::GetInstance()->IsConcurrentMarkingOrFinished());
ASSERT(valueRegion->InSharedSweepableSpace());
if (valueRegion->InSCollectSet() && objectRegion->InSharedHeap() && (valueRegion != objectRegion)) {
objectRegion->AtomicInsertCrossRegionRSet(slotAddr);
}
TaggedObject *heapValue = JSTaggedValue(value).GetHeapObject();
if (valueRegion->AtomicMark(heapValue)) {
std::atomic_thread_fence(std::memory_order_seq_cst);
Heap *heap = const_cast<Heap*>(thread->GetEcmaVM()->GetHeap());
MarkWorkNode *&localBuffer = heap->GetMarkingObjectLocalBuffer();
SharedHeap::GetInstance()->GetWorkManager()->PushToLocalMarkingBuffer(localBuffer, heapValue);
}
}
template <Region::RegionSpaceKind kind>
ARK_NOINLINE bool BatchBitSet([[maybe_unused]] const JSThread* thread, Region* objectRegion, JSTaggedValue* dst,
size_t count)
{
bool allValueNotHeap = true;
Region::Updater updater = objectRegion->GetBatchRSetUpdater<kind>(ToUintPtr(dst));
for (size_t i = 0; i < count; i++, updater.Next()) {
JSTaggedValue taggedValue = dst[i];
if (!taggedValue.IsHeapObject()) {
continue;
}
allValueNotHeap = false;
const Region* valueRegion = Region::ObjectAddressToRange(taggedValue.GetTaggedObject());
#if ECMASCRIPT_ENABLE_BARRIER_CHECK
ASSERT(taggedValue.GetRawData() != JSTaggedValue::VALUE_UNDEFINED);
if (!thread->GetEcmaVM()->GetHeap()->IsAlive(taggedValue.GetHeapObject())) {
LOG_FULL(FATAL) << "WriteBarrier checked value:" << taggedValue.GetRawData() << " is invalid!";
}
#endif
if (valueRegion->InSharedSweepableSpace()) {
#ifndef NDEBUG
if (UNLIKELY(taggedValue.IsWeakForHeapObject())) {
CHECK_NO_LOCAL_TO_SHARE_WEAK_REF_HANDLE;
}
#endif
updater.UpdateLocalToShare();
continue;
}
if constexpr (kind == Region::InGeneralOld) {
#if USE_STICKY_CMS_GC
if (!valueRegion->InSharedHeap() && taggedValue.GetTaggedObject()->IsInYoung()) {
updater.UpdateOldToNew();
continue;
}
#else
if (valueRegion->InYoungSpace()) {
updater.UpdateOldToNew();
continue;
}
#endif
}
}
return allValueNotHeap;
}
void Barriers::CMCWriteBarrier(const JSThread *thread, void *obj, size_t offset, JSTaggedType value)
{
ASSERT(g_isEnableCMCGC);
common::BaseRuntime::WriteBarrier(obj, (void *)((uintptr_t)obj + offset), (void*)value);
return;
}
#ifdef ARK_USE_SATB_BARRIER
void Barriers::CMCArrayCopyWriteBarrier(const JSThread *thread, const TaggedObject *dstObj, void* src, void* dst,
size_t count)
{
ASSERT(g_isEnableCMCGC);
JSTaggedType *dstPtr = reinterpret_cast<JSTaggedType *>(dst);
JSTaggedType *srcPtr = reinterpret_cast<JSTaggedType *>(src);
for (size_t i = 0; i < count; i++) {
JSTaggedType offset = i * sizeof(JSTaggedType);
JSTaggedType value = *reinterpret_cast<JSTaggedType *>(static_cast<JSTaggedType>(srcPtr) + offset);
void* obj = reinterpret_cast<void*>(const_cast<JSTaggedType *>(dstObj));
void* field = reinterpret_cast<void*>(static_cast<JSTaggedType>(dst) + offset);
common::BaseRuntime::WriteBarrier(obj, field, (void*)value);
}
return;
}
#else
bool Barriers::ShouldProcessSATB(common::GCPhase gcPhase)
{
switch (gcPhase) {
case common::GCPhase::GC_PHASE_ENUM:
case common::GCPhase::GC_PHASE_MARK:
case common::GCPhase::GC_PHASE_FINAL_MARK:
case common::GCPhase::GC_PHASE_REMARK_SATB:
return true;
default:
return false;
}
}
bool Barriers::ShouldGetGCReason(common::GCPhase gcPhase)
{
switch (gcPhase) {
case common::GCPhase::GC_PHASE_ENUM:
case common::GCPhase::GC_PHASE_MARK:
case common::GCPhase::GC_PHASE_POST_MARK:
return true;
default:
return false;
}
}
bool Barriers::ShouldUpdateRememberSet(common::GCPhase gcPhase)
{
if (common::Heap::GetHeap().GetGCReason() == common::GC_REASON_YOUNG || !ShouldGetGCReason(gcPhase)) {
return true;
}
return false;
}
void Barriers::CMCArrayCopyWriteBarrier(const JSThread *thread, const TaggedObject *dstObj, void* src, void* dst,
size_t count)
{
ASSERT(g_isEnableCMCGC);
ASSERT(dstObj != nullptr);
common::BaseObject* object = reinterpret_cast<BaseObject*>(const_cast<TaggedObject*>(dstObj));
common::RegionDesc::InlinedRegionMetaData *objMetaRegion =
common::RegionDesc::InlinedRegionMetaData::GetInlinedRegionMetaData(reinterpret_cast<uintptr_t>(object));
JSTaggedType *srcPtr = reinterpret_cast<JSTaggedType *>(src);
common::GCPhase gcPhase = thread->GetCMCGCPhase();
if (ShouldUpdateRememberSet(gcPhase) && !objMetaRegion->IsInYoungSpaceForWB()) {
auto checkReference = [&](BaseObject *ref) {
common::RegionDesc::InlinedRegionMetaData *refMetaRegion =
common::RegionDesc::InlinedRegionMetaData::GetInlinedRegionMetaData(reinterpret_cast<uintptr_t>(ref));
return refMetaRegion->IsInYoungSpaceForWB();
};
for (size_t i = 0; i < count; i++) {
JSTaggedType ref = *reinterpret_cast<JSTaggedType *>(ToUintPtr(srcPtr) + i * sizeof(JSTaggedType));
if (!common::Heap::IsTaggedObject(reinterpret_cast<common::HeapAddress>(ref))) {
continue;
}
ASSERT(common::Heap::IsHeapAddress(ref));
if (checkReference(reinterpret_cast<BaseObject *>(ref))) {
objMetaRegion->MarkRSetCardTable(object);
break;
}
}
}
if (ShouldProcessSATB(gcPhase)) {
common::Mutator* mutator = thread->GetMutator();
for (size_t i = 0; i < count; i++) {
JSTaggedType ref = *reinterpret_cast<JSTaggedType *>(ToUintPtr(srcPtr) + i * sizeof(JSTaggedType));
if (!common::Heap::IsTaggedObject(reinterpret_cast<common::HeapAddress>(ref))) {
continue;
}
ref = ref & ~(common::Barrier::TAG_WEAK);
mutator->RememberObjectInSatbBuffer(reinterpret_cast<BaseObject *>(ref));
}
}
}
#endif
void Barriers::CMCArrayCopyReadBarrierForward(const JSThread *thread, JSTaggedValue* dst, const JSTaggedValue* src,
size_t count)
{
for (size_t i = 0; i < count; i++) {
JSTaggedType valueToRef = Barriers::GetTaggedValue(thread, ToUintPtr(src) + i * sizeof(JSTaggedType));
Barriers::SetObject<false>(thread, dst, i * sizeof(JSTaggedType), valueToRef);
}
}
void Barriers::CMCArrayCopyReadBarrierBackward(const JSThread *thread, JSTaggedValue* dst, const JSTaggedValue* src,
size_t count)
{
for (size_t i = count; i > 0; i--) {
JSTaggedType valueToRef = Barriers::GetTaggedValue(thread, ToUintPtr(src) + (i - 1) * sizeof(JSTaggedType));
Barriers::SetObject<false>(thread, dst, (i - 1) * sizeof(JSTaggedType), valueToRef);
}
}
void Barriers::CheckObjectForCMS(const JSThread *thread, void *obj, size_t offset, JSTaggedValue value,
bool writeBarrierCheck)
{
Region *objectRegion = Region::ObjectAddressToRange(static_cast<TaggedObject *>(obj));
if (!objectRegion->InSharedHeap() && static_cast<TaggedObject *>(obj)->IsInvalid()) {
LOG_ECMA(FATAL) << "obj state is invalid";
}
if (thread->IsReadyToConcurrentMark() && objectRegion->InSlotSpace() &&
static_cast<TaggedObject *>(obj)->IsInOld() && !objectRegion->Test(obj)) {
LOG_ECMA(FATAL) << "obj is old but not marked";
}
if (value.IsHeapObject()) {
TaggedObject *heapValue = value.GetHeapObject();
Region *valueRegion = Region::ObjectAddressToRange(heapValue);
if (!valueRegion->InSharedHeap() && heapValue->IsInvalid()) {
LOG_ECMA(FATAL) << "value state is invalid";
}
if (thread->IsReadyToConcurrentMark()) {
if (valueRegion->InSlotSpace() && heapValue->IsInOld() && !valueRegion->Test(heapValue)) {
LOG_ECMA(FATAL) << "value is old but not marked";
}
if (!writeBarrierCheck && valueRegion->InSlotSpace() && static_cast<TaggedObject *>(obj)->IsInOld() &&
heapValue->IsInYoung() && !objectRegion->TestOldToNew(reinterpret_cast<uintptr_t>(obj) + offset)) {
LOG_ECMA(FATAL) << "old to new reference not marked in RSet";
}
}
}
}
void Barriers::CheckValueForCMS(const JSThread *thread, JSTaggedValue value)
{
if (value.IsHeapObject()) {
TaggedObject *heapValue = value.GetHeapObject();
Region *valueRegion = Region::ObjectAddressToRange(heapValue);
if (!valueRegion->InSharedHeap() && heapValue->IsInvalid()) {
LOG_ECMA(FATAL) << "value state is invalid";
}
if (thread->IsReadyToConcurrentMark()) {
if (valueRegion->InSlotSpace() && heapValue->IsInOld() && !valueRegion->Test(heapValue)) {
LOG_ECMA(FATAL) << "value is old but not marked";
}
}
}
}
#ifndef USE_CMC_GC
JSTaggedType ReadBarrierImpl(const JSThread *thread, uintptr_t slotAddress)
{
ASSERT(!thread->IsJitThread());
ASSERT(thread->NeedReadBarrier());
ObjectSlot slot(slotAddress);
JSTaggedValue value = slot.GetTaggedValue();
auto object = value.GetHeapObject();
if (UNLIKELY(object == nullptr)) {
return slot.GetTaggedType();
}
Region *objectRegion = Region::ObjectAddressToRange(object);
if (objectRegion->InSharedHeap()) {
return slot.GetTaggedType();
}
if (objectRegion->IsFromRegion()) {
MarkWord markWord(object, RELAXED_LOAD);
TaggedObject *toObject = nullptr;
if (markWord.IsForwardingAddress()) {
toObject = markWord.ToForwardingAddress();
} else {
toObject = thread->GetLocalCCEvacuator()->Copy(object, markWord);
}
if (value.IsWeak()) {
slot.UpdateWeak(ToUintPtr(toObject));
} else {
slot.Update(toObject);
}
return slot.GetTaggedType();
}
return slot.GetTaggedType();
}
JSTaggedType ReadBarrierForStringTableSlotImpl(JSTaggedType value)
{
if (value == reinterpret_cast<JSTaggedType>(nullptr)) {
return reinterpret_cast<JSTaggedType>(nullptr);
}
Region *objectRegion = Region::ObjectAddressToRange(value);
if (!objectRegion->InSharedSweepableSpace() || objectRegion->IsToRegion()) {
return value;
}
if (objectRegion->Test(value)) {
if (objectRegion->InSCollectSet()) {
MarkWord markWord(reinterpret_cast<TaggedObject *>(value), RELAXED_LOAD);
ASSERT(markWord.IsForwardingAddress());
return reinterpret_cast<JSTaggedType>(markWord.ToForwardingAddress());
}
return value;
}
return reinterpret_cast<JSTaggedType>(nullptr);
}
#endif
template bool BatchBitSet<Region::InYoung>(const JSThread*, Region*, JSTaggedValue*, size_t);
template bool BatchBitSet<Region::InGeneralOld>(const JSThread*, Region*, JSTaggedValue*, size_t);
template bool BatchBitSet<Region::Other>(const JSThread*, Region*, JSTaggedValue*, size_t);
}