* Copyright (c) 2025 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ecmascript/mem/cms_mem/slot_space-inl.h"
#include "ecmascript/mem/heap-inl.h"
namespace panda::ecmascript {
SlotSpace::SlotSpace(Heap *heap, size_t initialCapacity, size_t maximumCapacity)
: Space(heap, heap->GetHeapRegionAllocator(), MemSpaceType::SLOT_SPACE, initialCapacity, maximumCapacity),
SweepableSpace(),
localHeap_(heap),
minimumCapacity_(initialCapacity)
{
InitializeAllocators();
}
SlotSpace::~SlotSpace()
{
for (CMSRegionChainManager *regionChainManager : regionChainManagerInstances_) {
delete regionChainManager;
}
for (SlotAllocator *allocator : allocatorInstances_) {
delete allocator;
}
for (CMSRegionChainManager *gcRegionChainManager : gcRegionChainManagerInstances_) {
delete gcRegionChainManager;
}
for (SlotAllocator *gcAllocator : gcAllocatorInstances_) {
delete gcAllocator;
}
regionChainManagerInstances_.clear();
allocatorInstances_.clear();
gcRegionChainManagerInstances_.clear();
gcAllocatorInstances_.clear();
}
void SlotSpace::InitializeAllocators()
{
size_t preSize = 0;
for (size_t i = 0; i < SlotSpaceConfig::NUM_SLOTS; ++i) {
size_t slotSize = GetSlotSizeByIdx(i);
if (slotSize > preSize) {
SlotAllocator *allocator = new SlotAllocator();
SlotAllocator *gcAllocator = new SlotAllocator();
CMSRegionChainManager *regionChainManager = new CMSRegionChainManager();
CMSRegionChainManager *gcRegionChainManager = new CMSRegionChainManager();
allocator->Initialize(slotSize, regionChainManager);
gcAllocator->Initialize(slotSize, gcRegionChainManager);
regionChainManager->Initialize(localHeap_, slotSize);
gcRegionChainManager->Initialize(localHeap_, slotSize);
allocatorInstances_.emplace_back(allocator);
gcAllocatorInstances_.emplace_back(gcAllocator);
regionChainManagerInstances_.emplace_back(regionChainManager);
gcRegionChainManagerInstances_.emplace_back(gcRegionChainManager);
}
allocators_[i] = allocatorInstances_.back();
gcAllocators_[i] = gcAllocatorInstances_.back();
ASSERT(allocators_[i]->GetSlotSize() >= i * SlotSpaceConfig::SLOT_STEP_SIZE);
ASSERT(gcAllocators_[i]->GetSlotSize() >= i * SlotSpaceConfig::SLOT_STEP_SIZE);
preSize = slotSize;
}
}
void SlotSpace::TryTriggerGCIfNeed()
{
localHeap_->TryTriggerConcurrentMarking(MarkReason::ALLOCATION_LIMIT);
}
void SlotSpace::RequestGC()
{
if constexpr (G_USE_STICKY_CMS_GC) {
static int num = 0;
if (num++ % 2) {
localHeap_->CollectGarbage(TriggerGCType::CMS_GC, GCReason::ALLOCATION_FAILED);
} else {
localHeap_->CollectGarbage(TriggerGCType::STICKY_CMS_GC, GCReason::ALLOCATION_FAILED);
}
} else {
localHeap_->CollectGarbage(TriggerGCType::CMS_GC, GCReason::ALLOCATION_FAILED);
}
}
bool SlotSpace::TryExpandAllocator(SlotAllocator *allocator, MemoryCheckerKind checkerKind)
{
switch (checkerKind) {
case MemoryCheckerKind::HARD:
if (localHeap_->SlotSpaceExceedLimit()) {
return false;
}
break;
case MemoryCheckerKind::SOFT:
if (localHeap_->SlotSpaceExceedCapacity()) {
return false;
}
break;
default:
LOG_ECMA(FATAL) << "this branch is unreachable, " << static_cast<int>(checkerKind);
UNREACHABLE();
}
ExpandAllocator(allocator);
return true;
}
template<bool expandInGC>
void SlotSpace::ExpandAllocator(SlotAllocator *allocator)
{
Region *region = AllocateRegion(allocator->GetSlotSize());
allocator->Expand(region);
if constexpr(!expandInGC) {
IncreaseCommitted(region->GetCapacity());
} else {
region->SetRegionTypeFlag(RegionTypeFlag::TO);
region->CreateLocalToShareRememberedSet();
region->SwapLocalToShareRSetForCS();
}
}
Region *SlotSpace::ExpandRegionForSnapshot(size_t slotSize, size_t allocSize)
{
auto allocator = allocators_[GetSlotIdxBySize(slotSize)];
allocateAfterLastGC_ += allocator->Discard();
allocateAfterLastGC_ += allocSize;
Region *region = AllocateRegion(slotSize);
allocator->Expand(region, allocSize);
return region;
}
Region *SlotSpace::AllocateRegion(size_t slotSize)
{
Region *region = heapRegionAllocator_->AllocateAlignedRegion(this, DEFAULT_REGION_SIZE, localHeap_->GetJSThread(),
localHeap_, false, slotSize);
ASSERT(region != nullptr);
return region;
}
void SlotSpace::IterateOverObjects(const std::function<void(TaggedObject *object)> &visitor)
{
for (SlotAllocator *allocator : allocatorInstances_) {
allocator->Discard();
}
for (CMSRegionChainManager *regionChainManager : regionChainManagerInstances_) {
size_t slotSize = regionChainManager->GetSlotSize();
regionChainManager->EnumerateRegions([&visitor, slotSize](Region *region) {
ASSERT(slotSize > 0);
uintptr_t curPtr = region->GetBegin();
uintptr_t endPtr = CMSRegion::GetAllocatableEnd(CMSRegion::FromRegion(region), slotSize);
while (curPtr < endPtr) {
SlotFreeSegment *freeSegment = SlotFreeSegment::Cast(curPtr);
size_t objSize;
ASAN_UNPOISON_MEMORY_REGION(freeSegment, TaggedObject::TaggedObjectSize());
if (!freeSegment->IsSlotFreeSegment()) {
FreeObject *freeObject = FreeObject::Cast(curPtr);
ASAN_UNPOISON_MEMORY_REGION(freeObject, TaggedObject::TaggedObjectSize());
if (!freeObject->IsFreeObject()) {
auto obj = reinterpret_cast<TaggedObject *>(curPtr);
visitor(obj);
objSize = slotSize;
} else {
freeObject->AsanUnPoisonFreeObject();
ASSERT(freeObject->Available() <= slotSize);
objSize = slotSize;
freeObject->AsanPoisonFreeObject();
}
} else {
freeSegment->AsanUnPoisonFreeSegment();
objSize = freeSegment->GetSize();
freeSegment->AsanPoisonFreeSegment();
}
curPtr += objSize;
ASSERT(objSize % slotSize == 0);
CHECK_OBJECT_SIZE(objSize);
}
CHECK_REGION_END(curPtr, endPtr);
});
}
}
template <bool isSticky>
void SlotSpace::PrepareSweeping()
{
for (SlotAllocator *allocator : allocatorInstances_) {
allocateAfterLastGC_ += allocator->Discard();
}
size_t survivalObjectSize = 0;
for (CMSRegionChainManager *regionChainManager : regionChainManagerInstances_) {
survivalObjectSize += regionChainManager->PrepareSweeping<isSticky>(pendingReclaimFromRegions_);
}
SetSurvivalObjectSize(survivalObjectSize);
}
template void SlotSpace::PrepareSweeping<true>();
template void SlotSpace::PrepareSweeping<false>();
template <bool isSticky>
void SlotSpace::Sweep()
{
for (SlotAllocator *allocator : allocatorInstances_) {
allocateAfterLastGC_ += allocator->Discard();
}
size_t survivalObjectSize = 0;
for (CMSRegionChainManager *regionChainManager : regionChainManagerInstances_) {
survivalObjectSize += regionChainManager->Sweep<isSticky>(pendingReclaimFromRegions_);
}
SetSurvivalObjectSize(survivalObjectSize);
}
template void SlotSpace::Sweep<true>();
template void SlotSpace::Sweep<false>();
void SlotSpace::AsyncSweep([[maybe_unused]] bool isMain, [[maybe_unused]] bool releaseMemory)
{
for (CMSRegionChainManager *regionChainManager : regionChainManagerInstances_) {
regionChainManager->ConcurrentSweep(CMSRegionChainManager::SweepMode::SWEEP_ALL);
}
}
void SlotSpace::AddSweptRegionChainManager(CMSRegionChainManager *sweptRegionChainManager)
{
LockHolder holder(mutex_);
sweptRegionChainManagers_.emplace_back(sweptRegionChainManager);
}
bool SlotSpace::TryFillSweptRegion()
{
std::vector<CMSRegionChainManager *> sweptRegionChainManagers;
{
LockHolder holder(mutex_);
std::swap(sweptRegionChainManagers_, sweptRegionChainManagers);
}
for (CMSRegionChainManager *sweptRegionChainManager : sweptRegionChainManagers) {
sweptRegionChainManager->FinishSweeping();
}
return true;
}
bool SlotSpace::FinishFillSweptRegion()
{
for (CMSRegionChainManager *sweptRegionChainManager : sweptRegionChainManagers_) {
sweptRegionChainManager->FinishSweeping();
}
sweptRegionChainManagers_.clear();
return true;
}
void SlotSpace::PrepareCompact()
{
committedSize_ = 0;
ASSERT(pendingReclaimFromRegions_.empty());
pendingReclaimFromRegions_.reserve(GetRegionCount());
for (CMSRegionChainManager *regionChainManager : regionChainManagerInstances_) {
regionChainManager->PrepareCompact(pendingReclaimFromRegions_);
}
for (SlotAllocator *allocator : allocatorInstances_) {
allocateAfterLastGC_ += allocator->Discard();
}
}
void SlotSpace::ReclaimFromRegions(bool needCacheRegion)
{
if (!needCacheRegion) {
cacheRegionSize_ = 0;
}
for (Region *region : pendingReclaimFromRegions_) {
region->DeleteLocalToShareRSet();
heapRegionAllocator_->FreeRegion(region, cacheRegionSize_);
}
pendingReclaimFromRegions_.clear();
}
void SlotSpace::MergeToRegions()
{
for (size_t i = 0; i < gcRegionChainManagerInstances_.size(); i++) {
CMSRegionChainManager *gcRegionChainManager = gcRegionChainManagerInstances_[i];
if (!gcRegionChainManager->IsEmpty()) {
CMSRegionChainManager *regionChainManager = regionChainManagerInstances_[i];
gcRegionChainManager->EnumerateRegions([this, regionChainManager](Region *toRegion) {
toRegion->ResetSwept();
toRegion->MergeLocalToShareRSetForCS();
regionChainManager->AddNewRegion(toRegion);
IncreaseCommitted(toRegion->GetCapacity());
});
gcRegionChainManager->ClearRegionList();
}
}
for (SlotAllocator *allocator : gcAllocatorInstances_) {
allocator->Discard();
}
}
void SlotSpace::AdjustCapacity(JSThread *thread)
{
allocateAfterLastGC_ = 0;
size_t objectSize = GetHeapObjectSize() + localHeap_->GetHugeObjectSpace()->GetHeapObjectSize();
double objectSurvivalRate = static_cast<double>(objectSize) / initialCapacity_;
double allocSpeed = localHeap_->GetMemController()->GetSlotAndHugeSpaceAllocationThroughputPerMS();
size_t newCapacity = std::min(maximumCapacity_,
committedSize_ + localHeap_->GetEcmaVM()->GetEcmaParamConfiguration().GetMinGrowingStep());
initialCapacity_ = std::min(initialCapacity_, committedSize_);
if (objectSurvivalRate > GROW_OBJECT_SURVIVAL_RATE) {
newCapacity = std::max(newCapacity, initialCapacity_ * GROWING_FACTOR);
while (committedSize_ >= newCapacity && newCapacity < maximumCapacity_) {
newCapacity = newCapacity * GROWING_FACTOR;
}
newCapacity = std::min(newCapacity, maximumCapacity_);
ASSERT(newCapacity >= committedSize_);
cacheRegionSize_ = newCapacity - committedSize_;
SetInitialCapacity(newCapacity);
return;
}
if (initialCapacity_ < (MIN_GC_INTERVAL_MS * allocSpeed) && initialCapacity_ < maximumCapacity_) {
newCapacity = std::max(newCapacity, initialCapacity_ * GROWING_FACTOR);
newCapacity = std::min(newCapacity, maximumCapacity_);
ASSERT(newCapacity >= committedSize_);
cacheRegionSize_ = newCapacity - committedSize_;
SetInitialCapacity(newCapacity);
return;
}
ASSERT(newCapacity >= committedSize_);
cacheRegionSize_ = newCapacity - committedSize_;
SetInitialCapacity(newCapacity);
}
template void SlotSpace::ExpandAllocator<true>(SlotAllocator *allocator);
template void SlotSpace::ExpandAllocator<false>(SlotAllocator *allocator);
}