* Copyright (c) 2025 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ecmascript/mem/shared_heap/shared_gc_evacuator.h"
#include "common_components/taskpool/taskpool.h"
#include "ecmascript/mem/object_xray.h"
#include "ecmascript/mem/tlab_allocator-inl.h"
namespace panda::ecmascript {
void SharedGCEvacuator::Evacuate()
{
EvacuateRegions();
UpdateReference();
}
void SharedGCEvacuator::EvacuateRegionWorkload::Process(uint32_t threadIndex)
{
ECMA_BYTRACE_NAME(HITRACE_LEVEL_COMMERCIAL, HITRACE_TAG_ARK, "SharedGCEvacuator::EvacuateRegionWorkload", "");
auto sTlabAllocator = evacuator_->GetOrCreateTlab(threadIndex);
region_->IterateAllMarkedBits([this, sTlabAllocator](void *mem) {
auto header = reinterpret_cast<TaggedObject *>(mem);
JSHClass *klass = header->GetClass();
size_t size = header->GetSize();
uintptr_t address = sTlabAllocator->Allocate(size, SHARED_COMPRESS_SPACE);
ASSERT(address != 0);
if (memcpy_s(ToVoidPtr(address), size, ToVoidPtr(ToUintPtr(mem)), size) != EOK) {
LOG_ECMA_MEM(FATAL) << "memcpy_s failed";
UNREACHABLE();
}
if (UNLIKELY(inHeapProfiler_)) {
sHeap_->OnMoveEvent(reinterpret_cast<intptr_t>(mem), reinterpret_cast<TaggedObject *>(address), size);
}
Barriers::SetPrimitive(header, 0, MarkWord::FromForwardingAddress(address));
});
}
void SharedGCEvacuator::EvacuateRegions()
{
TRACE_GC(GCStats::Scope::ScopeId::Evacuate, sHeap_->GetEcmaGCStats());
ECMA_BYTRACE_NAME(HITRACE_LEVEL_COMMERCIAL, HITRACE_TAG_ARK, ("SharedGCEvacuator::EvacuateRegions;cset count: "
+ std::to_string(sHeap_->GetOldSpace()->GetCollectSetRegionCount())).c_str(), "");
bool inHeapProfiler = sHeap_->InHeapProfiler();
sHeap_->GetOldSpace()->EnumerateCollectRegionSet([this, inHeapProfiler](Region *region) {
ASSERT(region->InSCollectSet());
AddWorkload(std::make_unique<EvacuateRegionWorkload>(this, region, sHeap_, inHeapProfiler));
});
PostParallelTasks();
ProcessWorkloads(MAIN_THREAD_INDEX);
for (auto sTlabAllocator : sTlabs_) {
if (sTlabAllocator) {
sTlabAllocator->Finalize();
delete sTlabAllocator;
}
}
sTlabs_.fill(nullptr);
}
void SharedGCEvacuator::UpdateReference()
{
TRACE_GC(GCStats::Scope::ScopeId::UpdateReference, sHeap_->GetEcmaGCStats());
ECMA_BYTRACE_NAME(HITRACE_LEVEL_COMMERCIAL, HITRACE_TAG_ARK, "SharedGCEvacuator::UpdateReference", "");
Runtime *runtime = Runtime::GetInstance();
runtime->GCIterateThreadList([this](JSThread *thread) {
ASSERT(thread->IsSuspended() || thread->HasLaunchedSuspendAll());
auto heap = const_cast<Heap*>(thread->GetEcmaVM()->GetHeap());
heap->GetSweeper()->EnsureAllTaskFinished();
heap->EnumerateRegions([this](Region *region) {
AddWorkload(std::make_unique<UpdateLocalReferenceWorkload>(this, region));
});
});
sHeap_->EnumerateOldSpaceRegions([this](Region *region) {
ASSERT(!region->InSCollectSet());
if (region->IsToRegion()) {
AddWorkload(std::make_unique<UpdateToRegionReferenceWorkload>(this, region));
} else {
AddWorkload(std::make_unique<UpdateSharedReferenceWorkload>(this, region));
}
});
PostParallelTasks();
runtime->IterateSharedRoot(rootVisitor_);
runtime->GCIterateThreadList([this](JSThread *thread) {
ASSERT(thread->IsSuspended() || thread->HasLaunchedSuspendAll());
auto vm = thread->GetEcmaVM();
ObjectXRay::VisitVMRoots(vm, rootVisitor_);
});
ProcessWorkloads(MAIN_THREAD_INDEX);
}
void SharedGCEvacuator::UpdateLocalReferenceWorkload::Process([[maybe_unused]]uint32_t threadIndex)
{
region_->IterateAllLocalToShareBits([this](void *mem) {
ObjectSlot slot(ToUintPtr(mem));
return evacuator_->UpdateObjectSlot(slot);
});
}
void SharedGCEvacuator::UpdateSharedReferenceWorkload::Process([[maybe_unused]]uint32_t threadIndex)
{
region_->IterateAllCrossRegionBits([this](void *mem) {
ObjectSlot slot(ToUintPtr(mem));
evacuator_->UpdateObjectSlot(slot);
});
}
void SharedGCEvacuator::UpdateToRegionReferenceWorkload::Process([[maybe_unused]]uint32_t threadIndex)
{
uintptr_t curPtr = region_->GetBegin();
uintptr_t endPtr = region_->GetEnd();
size_t objSize = 0;
while (curPtr < endPtr) {
auto freeObject = FreeObject::Cast(curPtr);
if (!freeObject->IsFreeObject()) {
auto obj = reinterpret_cast<TaggedObject *>(curPtr);
evacuator_->ProcessObjectField(obj, obj->GetClass());
objSize = obj->GetSize();
} else {
objSize = freeObject->Available();
}
curPtr += objSize;
CHECK_OBJECT_SIZE(objSize);
}
CHECK_REGION_END(curPtr, endPtr);
}
bool SharedGCEvacuator::ParallelTask::Run([[maybe_unused]]uint32_t threadIndex)
{
ECMA_BYTRACE_NAME(HITRACE_LEVEL_COMMERCIAL, HITRACE_TAG_ARK, "SharedGCEvacuator::ParallelTask", "");
evacuator_->ProcessWorkloads(threadIndex);
return true;
}
void SharedGCEvacuator::WaitFinished()
{
if (parallel_ > 0) {
LockHolder holder(lock_);
while (parallel_ > 0) {
condition_.Wait(&lock_);
}
}
}
int SharedGCEvacuator::CalculateParallelThreadNum()
{
constexpr uint32_t regionPerThread = 8;
uint32_t count = workloads_.size() / regionPerThread;
return static_cast<int>(std::min(std::max(1U, count), common::Taskpool::GetCurrentTaskpool()->GetTotalThreadNum()));
}
bool SharedGCEvacuator::UpdateObjectSlot(ObjectSlot slot)
{
JSTaggedValue value(slot.GetTaggedType());
if (!value.IsHeapObject()) {
return false;
}
Region *region = Region::ObjectAddressToRange(slot.GetTaggedType());
if (!region->InSCollectSet()) {
return true;
}
ASSERT(region->InSharedHeap());
bool isWeak = value.IsWeakForHeapObject();
TaggedObject *object = value.GetHeapObject();
MarkWord markWord(object, RELAXED_LOAD);
ASSERT(markWord.IsForwardingAddress());
TaggedObject *dst = markWord.ToForwardingAddress();
if (isWeak) {
dst = JSTaggedValue(dst).CreateAndGetWeakRef().GetRawTaggedObject();
}
slot.Update(dst);
return true;
}
void SharedGCEvacuator::ObjectFieldCSetVisitor::VisitObjectRangeImpl(BaseObject *root, uintptr_t startAddr,
uintptr_t endAddr, VisitObjectArea area)
{
ObjectSlot start(startAddr);
ObjectSlot end(endAddr);
if (UNLIKELY(area == VisitObjectArea::IN_OBJECT)) {
JSHClass *hclass = TaggedObject::Cast(root)->GetClass();
ASSERT(!hclass->IsAllTaggedProp());
int index = 0;
TaggedObject *dst = hclass->GetLayout(THREAD_ARG_PLACEHOLDER).GetTaggedObject();
LayoutInfo *layout = LayoutInfo::UncheckCast(dst);
ObjectSlot realEnd = start;
realEnd += layout->GetPropertiesCapacity();
end = end > realEnd ? realEnd : end;
for (ObjectSlot slot = start; slot < end; slot++) {
auto attr = layout->GetAttr(THREAD_ARG_PLACEHOLDER, index++);
if (attr.IsTaggedRep()) {
evacuator_->UpdateObjectSlot(slot);
}
}
return;
}
for (ObjectSlot slot = start; slot < end; slot++) {
evacuator_->UpdateObjectSlot(slot);
}
}
void SharedGCEvacuator::ProcessObjectField(TaggedObject *object, JSHClass *hclass)
{
ObjectXRay::VisitObjectBody<VisitType::OLD_GC_VISIT>(object, hclass, objectFieldCSetVisitor_);
}
void SharedGCEvacuator::UpdateRootVisitor::VisitRoot([[maybe_unused]] Root type, ObjectSlot slot)
{
UpdateObjectSlotRoot(slot);
}
void SharedGCEvacuator::UpdateRootVisitor::VisitRangeRoot([[maybe_unused]] Root type, ObjectSlot start, ObjectSlot end)
{
for (ObjectSlot slot = start; slot < end; slot++) {
UpdateObjectSlotRoot(slot);
}
}
void SharedGCEvacuator::UpdateRootVisitor::VisitBaseAndDerivedRoot([[maybe_unused]] Root type, ObjectSlot base,
ObjectSlot derived, uintptr_t baseOldObject)
{
if (JSTaggedValue(base.GetTaggedType()).IsHeapObject()) {
derived.Update(base.GetTaggedType() + derived.GetTaggedType() - baseOldObject);
}
}
void SharedGCEvacuator::UpdateRootVisitor::UpdateObjectSlotRoot(ObjectSlot slot)
{
JSTaggedValue value(slot.GetTaggedType());
if (value.IsHeapObject()) {
Region *region = Region::ObjectAddressToRange(slot.GetTaggedType());
if (region->InSCollectSet()) {
ASSERT(region->InSharedHeap());
ASSERT(!value.IsWeakForHeapObject());
TaggedObject *object = value.GetHeapObject();
MarkWord markWord(object, RELAXED_LOAD);
if (markWord.IsForwardingAddress()) {
TaggedObject *dst = markWord.ToForwardingAddress();
slot.Update(dst);
} else {
slot.Clear();
}
}
}
}
void SharedGCEvacuator::PostParallelTasks()
{
if (sHeap_->IsParallelGCEnabled()) {
LockHolder holder(lock_);
parallel_ = CalculateParallelThreadNum();
auto dTid = DaemonThread::GetInstance()->GetThreadId();
for (int i = 0; i < parallel_; i++) {
common::Taskpool::GetCurrentTaskpool()->PostTask(std::make_unique<ParallelTask>(dTid, this));
}
}
}
}
