#ifndef V8_OBJECTS_FEEDBACK_VECTOR_INL_H_
#define V8_OBJECTS_FEEDBACK_VECTOR_INL_H_
#include "src/objects/feedback-vector.h"
#include <optional>
#include "src/common/globals.h"
#include "src/heap/heap-write-barrier-inl.h"
#include "src/objects/code-inl.h"
#include "src/objects/feedback-cell-inl.h"
#include "src/objects/maybe-object-inl.h"
#include "src/objects/shared-function-info.h"
#include "src/objects/smi.h"
#include "src/objects/tagged.h"
#include "src/roots/roots-inl.h"
#include "src/torque/runtime-macro-shims.h"
#include "src/torque/runtime-support.h"
#include "src/objects/object-macros.h"
namespace v8::internal {
#include "torque-generated/src/objects/feedback-vector-tq-inl.inc"
TQ_OBJECT_CONSTRUCTORS_IMPL(FeedbackVector)
OBJECT_CONSTRUCTORS_IMPL(FeedbackMetadata, HeapObject)
INT32_ACCESSORS(FeedbackMetadata, slot_count, kSlotCountOffset)
INT32_ACCESSORS(FeedbackMetadata, create_closure_slot_count,
kCreateClosureSlotCountOffset)
#define ASSERT_BUILTIN_ID_CONSECUTIVE(V, Location, Representation, Kind, \
Index) \
static_assert( \
static_cast<intptr_t>( \
Builtin::kLoadIC##Location##Representation##Kind##Baseline) + \
1 + Index == \
static_cast<intptr_t>( \
Builtin::kLoadIC##Location##Representation##Kind##Index##Baseline));
int32_t FeedbackMetadata::slot_count(AcquireLoadTag) const {
return ACQUIRE_READ_INT32_FIELD(*this, kSlotCountOffset);
}
int32_t FeedbackMetadata::create_closure_slot_count(AcquireLoadTag) const {
return ACQUIRE_READ_INT32_FIELD(*this, kCreateClosureSlotCountOffset);
}
int32_t FeedbackMetadata::get(int index) const {
CHECK_LT(static_cast<unsigned>(index), static_cast<unsigned>(word_count()));
int offset = kHeaderSize + index * kInt32Size;
return ReadField<int32_t>(offset);
}
void FeedbackMetadata::set(int index, int32_t value) {
DCHECK_LT(static_cast<unsigned>(index), static_cast<unsigned>(word_count()));
int offset = kHeaderSize + index * kInt32Size;
WriteField<int32_t>(offset, value);
}
bool FeedbackMetadata::is_empty() const {
DCHECK_IMPLIES(slot_count() == 0, create_closure_slot_count() == 0);
return slot_count() == 0;
}
int FeedbackMetadata::AllocatedSize() {
return SizeFor(slot_count(kAcquireLoad),
create_closure_slot_count(kAcquireLoad));
}
int FeedbackMetadata::word_count() const {
return FeedbackMetadata::word_count(slot_count());
}
int FeedbackMetadata::GetSlotSize(FeedbackSlotKind kind) {
switch (kind) {
case FeedbackSlotKind::kForIn:
case FeedbackSlotKind::kInstanceOf:
case FeedbackSlotKind::kTypeOf:
case FeedbackSlotKind::kCompareOp:
case FeedbackSlotKind::kBinaryOp:
case FeedbackSlotKind::kLiteral:
case FeedbackSlotKind::kJumpLoop:
return 1;
case FeedbackSlotKind::kCall:
case FeedbackSlotKind::kCloneObject:
case FeedbackSlotKind::kLoadProperty:
case FeedbackSlotKind::kLoadGlobalInsideTypeof:
case FeedbackSlotKind::kLoadGlobalNotInsideTypeof:
case FeedbackSlotKind::kLoadKeyed:
case FeedbackSlotKind::kHasKeyed:
case FeedbackSlotKind::kSetNamedSloppy:
case FeedbackSlotKind::kSetNamedStrict:
case FeedbackSlotKind::kDefineNamedOwn:
case FeedbackSlotKind::kDefineKeyedOwn:
case FeedbackSlotKind::kStoreGlobalSloppy:
case FeedbackSlotKind::kStoreGlobalStrict:
case FeedbackSlotKind::kSetKeyedSloppy:
case FeedbackSlotKind::kSetKeyedStrict:
case FeedbackSlotKind::kStoreInArrayLiteral:
case FeedbackSlotKind::kDefineKeyedOwnPropertyInLiteral:
case FeedbackSlotKind::kStringAddAndInternalize:
return 2;
case FeedbackSlotKind::kInvalid:
UNREACHABLE();
}
UNREACHABLE();
}
bool FeedbackVector::is_empty() const { return length() == 0; }
DEF_GETTER(FeedbackVector, has_metadata, bool) {
return shared_function_info()->HasFeedbackMetadata();
}
DEF_GETTER(FeedbackVector, metadata, Tagged<FeedbackMetadata>) {
return shared_function_info(cage_base)->feedback_metadata(cage_base);
}
DEF_ACQUIRE_GETTER(FeedbackVector, metadata, Tagged<FeedbackMetadata>) {
return shared_function_info(cage_base)->feedback_metadata(cage_base,
kAcquireLoad);
}
RELAXED_INT32_ACCESSORS(FeedbackVector, invocation_count,
kInvocationCountOffset)
void FeedbackVector::clear_invocation_count(RelaxedStoreTag tag) {
set_invocation_count(0, tag);
}
RELAXED_UINT8_ACCESSORS(FeedbackVector, invocation_count_before_stable,
kInvocationCountBeforeStableOffset)
int FeedbackVector::osr_urgency() const {
return OsrUrgencyBits::decode(osr_state());
}
void FeedbackVector::set_osr_urgency(int urgency) {
DCHECK(0 <= urgency && urgency <= FeedbackVector::kMaxOsrUrgency);
static_assert(FeedbackVector::kMaxOsrUrgency <= OsrUrgencyBits::kMax);
set_osr_state(OsrUrgencyBits::update(osr_state(), urgency));
}
void FeedbackVector::reset_osr_urgency() { set_osr_urgency(0); }
void FeedbackVector::RequestOsrAtNextOpportunity() {
set_osr_urgency(kMaxOsrUrgency);
}
void FeedbackVector::reset_osr_state() { set_osr_state(0); }
bool FeedbackVector::maybe_has_optimized_osr_code() const {
return maybe_has_maglev_osr_code() || maybe_has_turbofan_osr_code();
}
bool FeedbackVector::maybe_has_maglev_osr_code() const {
return MaybeHasMaglevOsrCodeBit::decode(osr_state());
}
bool FeedbackVector::maybe_has_turbofan_osr_code() const {
return MaybeHasTurbofanOsrCodeBit::decode(osr_state());
}
void FeedbackVector::set_maybe_has_optimized_osr_code(bool value,
CodeKind code_kind) {
if (code_kind == CodeKind::MAGLEV) {
CHECK(v8_flags.maglev_osr);
set_osr_state(MaybeHasMaglevOsrCodeBit::update(osr_state(), value));
} else {
CHECK_EQ(code_kind, CodeKind::TURBOFAN_JS);
set_osr_state(MaybeHasTurbofanOsrCodeBit::update(osr_state(), value));
}
}
bool FeedbackVector::interrupt_budget_reset_by_ic_change() const {
return InterruptBudgetResetByIcChangeBit::decode(flags());
}
void FeedbackVector::set_interrupt_budget_reset_by_ic_change(bool value) {
set_flags(InterruptBudgetResetByIcChangeBit::update(flags(), value));
}
bool FeedbackVector::was_once_deoptimized() const {
return invocation_count_before_stable(kRelaxedLoad) ==
kInvocationCountBeforeStableDeoptSentinel;
}
void FeedbackVector::set_was_once_deoptimized() {
set_invocation_count_before_stable(kInvocationCountBeforeStableDeoptSentinel,
kRelaxedStore);
}
bool FeedbackVector::tiering_in_progress() const {
return TieringInProgressBit::decode(flags());
}
std::optional<Tagged<Code>> FeedbackVector::GetOptimizedOsrCode(
Isolate* isolate, Handle<BytecodeArray> bytecode, FeedbackSlot slot) {
Tagged<MaybeObject> maybe_code = Get(isolate, slot);
if (maybe_code.IsCleared()) return {};
Tagged<Code> code =
Cast<CodeWrapper>(maybe_code.GetHeapObject())->code(isolate);
if (code->marked_for_deoptimization()) {
Set(slot, ClearedValue());
if (!bytecode.is_null()) {
RecomputeOptimizedOsrCodeFlags(isolate, bytecode);
}
return {};
}
return code;
}
void FeedbackVector::RecomputeOptimizedOsrCodeFlags(
Isolate* isolate, Handle<BytecodeArray> bytecode_array) {
bool turbofan = false;
bool maglev = false;
interpreter::BytecodeArrayIterator it(bytecode_array);
for (; !it.done(); it.Advance()) {
if (it.current_bytecode() != interpreter::Bytecode::kJumpLoop) continue;
if (auto code = GetOptimizedOsrCode(isolate, {}, it.GetSlotOperand(2))) {
if ((*code)->marked_for_deoptimization()) continue;
turbofan |= (*code)->is_turbofanned();
maglev |= (*code)->is_maglevved();
}
}
if (!maglev && maybe_has_maglev_osr_code()) {
set_maybe_has_optimized_osr_code(false, CodeKind::MAGLEV);
}
if (!turbofan && maybe_has_turbofan_osr_code()) {
set_maybe_has_optimized_osr_code(false, CodeKind::TURBOFAN_JS);
}
}
FeedbackSlot FeedbackVector::ToSlot(intptr_t index) {
if (index == static_cast<intptr_t>(FeedbackSlot::Invalid().ToInt())) {
return FeedbackSlot();
}
DCHECK_LE(static_cast<uintptr_t>(index),
static_cast<uintptr_t>(std::numeric_limits<int>::max()));
return FeedbackSlot(static_cast<int>(index));
}
#ifdef DEBUG
bool FeedbackVector::IsOfLegacyType(Tagged<MaybeObject> value) {
Tagged<HeapObject> heap_object;
if (value.GetHeapObject(&heap_object)) {
return IsFixedArray(heap_object) && !IsHashTable(heap_object);
}
return false;
}
#endif
Tagged<MaybeObject> FeedbackVector::Get(FeedbackSlot slot) const {
Tagged<MaybeObject> value = raw_feedback_slots(GetIndex(slot), kRelaxedLoad);
DCHECK(!IsOfLegacyType(value));
return value;
}
Tagged<MaybeObject> FeedbackVector::Get(PtrComprCageBase cage_base,
FeedbackSlot slot) const {
Tagged<MaybeObject> value =
raw_feedback_slots(cage_base, GetIndex(slot), kRelaxedLoad);
DCHECK(!IsOfLegacyType(value));
return value;
}
DirectHandle<FeedbackCell> FeedbackVector::GetClosureFeedbackCell(
Isolate* isolate, int index) const {
DCHECK_GE(index, 0);
return direct_handle(closure_feedback_cell_array()->get(index), isolate);
}
Tagged<FeedbackCell> FeedbackVector::closure_feedback_cell(int index) const {
DCHECK_GE(index, 0);
return closure_feedback_cell_array()->get(index);
}
Tagged<MaybeObject> FeedbackVector::SynchronizedGet(FeedbackSlot slot) const {
const int i = slot.ToInt();
DCHECK_LT(static_cast<unsigned>(i), static_cast<unsigned>(this->length()));
const int offset = kRawFeedbackSlotsOffset + i * kTaggedSize;
Tagged<MaybeObject> value =
TaggedField<MaybeObject>::Acquire_Load(*this, offset);
DCHECK(!IsOfLegacyType(value));
return value;
}
void FeedbackVector::SynchronizedSet(FeedbackSlot slot,
Tagged<MaybeObject> value,
WriteBarrierMode mode) {
DCHECK(!IsOfLegacyType(value));
const int i = slot.ToInt();
DCHECK_LT(static_cast<unsigned>(i), static_cast<unsigned>(this->length()));
const int offset = kRawFeedbackSlotsOffset + i * kTaggedSize;
TaggedField<MaybeObject>::Release_Store(*this, offset, value);
CONDITIONAL_WRITE_BARRIER(*this, offset, value, mode);
}
void FeedbackVector::Set(FeedbackSlot slot, Tagged<MaybeObject> value,
WriteBarrierMode mode) {
DCHECK(!IsOfLegacyType(value));
set_raw_feedback_slots(GetIndex(slot), value, kRelaxedStore, mode);
}
inline MaybeObjectSlot FeedbackVector::slots_start() {
return RawMaybeWeakField(OffsetOfElementAt(0));
}
BinaryOperationHint BinaryOperationHintFromFeedback(int type_feedback) {
switch (type_feedback) {
case BinaryOperationFeedback::kNone:
return BinaryOperationHint::kNone;
case BinaryOperationFeedback::kSignedSmall:
return BinaryOperationHint::kSignedSmall;
case BinaryOperationFeedback::kSignedSmallInputs:
return BinaryOperationHint::kSignedSmallInputs;
case BinaryOperationFeedback::kAdditiveSafeInteger:
return BinaryOperationHint::kAdditiveSafeInteger;
case BinaryOperationFeedback::kNumber:
return BinaryOperationHint::kNumber;
case BinaryOperationFeedback::kNumberOrOddball:
return BinaryOperationHint::kNumberOrOddball;
case BinaryOperationFeedback::kString:
return BinaryOperationHint::kString;
case BinaryOperationFeedback::kStringOrStringWrapper:
return BinaryOperationHint::kStringOrStringWrapper;
case BinaryOperationFeedback::kBigInt:
return BinaryOperationHint::kBigInt;
case BinaryOperationFeedback::kBigInt64:
return BinaryOperationHint::kBigInt64;
default:
return BinaryOperationHint::kAny;
}
UNREACHABLE();
}
template <CompareOperationFeedback::Type Feedback>
bool Is(int type_feedback) {
return !(type_feedback & ~Feedback);
}
CompareOperationHint CompareOperationHintFromFeedback(int type_feedback) {
if (Is<CompareOperationFeedback::kNone>(type_feedback)) {
return CompareOperationHint::kNone;
}
if (Is<CompareOperationFeedback::kSignedSmall>(type_feedback)) {
return CompareOperationHint::kSignedSmall;
} else if (Is<CompareOperationFeedback::kNumber>(type_feedback)) {
return CompareOperationHint::kNumber;
} else if (Is<CompareOperationFeedback::kNumberOrBoolean>(type_feedback)) {
return CompareOperationHint::kNumberOrBoolean;
} else if (Is<CompareOperationFeedback::kNumberOrOddball>(type_feedback)) {
return CompareOperationHint::kNumberOrOddball;
}
if (Is<CompareOperationFeedback::kInternalizedString>(type_feedback)) {
return CompareOperationHint::kInternalizedString;
} else if (Is<CompareOperationFeedback::kString>(type_feedback)) {
return CompareOperationHint::kString;
} else if (Is<CompareOperationFeedback::kStringOrOddball>(type_feedback) &&
!Is<CompareOperationFeedback::kOddball>(type_feedback)) {
return CompareOperationHint::kStringOrOddball;
}
if (Is<CompareOperationFeedback::kReceiver>(type_feedback)) {
return CompareOperationHint::kReceiver;
} else if (Is<CompareOperationFeedback::kReceiverOrNullOrUndefined>(
type_feedback)) {
return CompareOperationHint::kReceiverOrNullOrUndefined;
}
if (Is<CompareOperationFeedback::kBigInt64>(type_feedback)) {
return CompareOperationHint::kBigInt64;
} else if (Is<CompareOperationFeedback::kBigInt>(type_feedback)) {
return CompareOperationHint::kBigInt;
}
if (Is<CompareOperationFeedback::kSymbol>(type_feedback)) {
return CompareOperationHint::kSymbol;
}
DCHECK(Is<CompareOperationFeedback::kAny>(type_feedback));
return CompareOperationHint::kAny;
}
ForInHint ForInHintFromFeedback(ForInFeedback type_feedback) {
switch (type_feedback) {
case ForInFeedback::kNone:
return ForInHint::kNone;
case ForInFeedback::kEnumCacheKeys:
return ForInHint::kEnumCacheKeys;
case ForInFeedback::kEnumCacheKeysAndIndices:
return ForInHint::kEnumCacheKeysAndIndices;
default:
return ForInHint::kAny;
}
UNREACHABLE();
}
DirectHandle<Symbol> FeedbackVector::UninitializedSentinel(Isolate* isolate) {
return isolate->factory()->uninitialized_symbol();
}
Handle<Symbol> FeedbackVector::MegamorphicSentinel(Isolate* isolate) {
return isolate->factory()->megamorphic_symbol();
}
DirectHandle<Symbol> FeedbackVector::MegaDOMSentinel(Isolate* isolate) {
return isolate->factory()->mega_dom_symbol();
}
Tagged<Symbol> FeedbackVector::RawUninitializedSentinel(Isolate* isolate) {
return ReadOnlyRoots(isolate).uninitialized_symbol();
}
bool FeedbackMetadataIterator::HasNext() const {
return next_slot_.ToInt() < metadata()->slot_count();
}
FeedbackSlot FeedbackMetadataIterator::Next() {
DCHECK(HasNext());
cur_slot_ = next_slot_;
slot_kind_ = metadata()->GetKind(cur_slot_);
next_slot_ = FeedbackSlot(next_slot_.ToInt() + entry_size());
return cur_slot_;
}
int FeedbackMetadataIterator::entry_size() const {
return FeedbackMetadata::GetSlotSize(kind());
}
template <typename T>
Handle<T> NexusConfig::NewHandle(Tagged<T> object) const {
if (mode() == Mode::MainThread) {
return handle(object, isolate_);
}
DCHECK_EQ(mode(), Mode::BackgroundThread);
return handle(object, local_heap_);
}
Tagged<MaybeObject> NexusConfig::GetFeedback(Tagged<FeedbackVector> vector,
FeedbackSlot slot) const {
return vector->SynchronizedGet(slot);
}
void NexusConfig::SetFeedback(Tagged<FeedbackVector> vector, FeedbackSlot slot,
Tagged<MaybeObject> feedback,
WriteBarrierMode mode) const {
DCHECK(can_write());
vector->SynchronizedSet(slot, feedback, mode);
}
Tagged<MaybeObject> FeedbackNexus::UninitializedSentinel() const {
return *FeedbackVector::UninitializedSentinel(config()->isolate());
}
Tagged<MaybeObject> FeedbackNexus::MegamorphicSentinel() const {
return *FeedbackVector::MegamorphicSentinel(config()->isolate());
}
Tagged<MaybeObject> FeedbackNexus::MegaDOMSentinel() const {
return *FeedbackVector::MegaDOMSentinel(config()->isolate());
}
Tagged<MaybeObject> FeedbackNexus::FromHandle(
MaybeObjectDirectHandle slot) const {
return slot.is_null() ? ClearedValue() : *slot;
}
MaybeObjectHandle FeedbackNexus::ToHandle(Tagged<MaybeObject> value) const {
return value.IsCleared() ? MaybeObjectHandle()
: MaybeObjectHandle(config()->NewHandle(value));
}
Tagged<MaybeObject> FeedbackNexus::GetFeedback() const {
auto pair = GetFeedbackPair();
return pair.first;
}
Tagged<MaybeObject> FeedbackNexus::GetFeedbackExtra() const {
auto pair = GetFeedbackPair();
return pair.second;
}
std::pair<Tagged<MaybeObject>, Tagged<MaybeObject>>
FeedbackNexus::GetFeedbackPair() const {
if (config()->mode() == NexusConfig::BackgroundThread &&
feedback_cache_.has_value()) {
return std::make_pair(FromHandle(feedback_cache_->first),
FromHandle(feedback_cache_->second));
}
auto pair = FeedbackMetadata::GetSlotSize(kind()) == 2
? config()->GetFeedbackPair(vector(), slot())
: std::make_pair(config()->GetFeedback(vector(), slot()),
Tagged<MaybeObject>());
if (config()->mode() == NexusConfig::BackgroundThread &&
!feedback_cache_.has_value()) {
feedback_cache_ =
std::make_pair(ToHandle(pair.first), ToHandle(pair.second));
}
return pair;
}
template <typename FeedbackType>
void FeedbackNexus::SetFeedback(Tagged<FeedbackType> feedback,
WriteBarrierMode mode) {
config()->SetFeedback(vector(), slot(), feedback, mode);
}
template <typename FeedbackType, typename FeedbackExtraType>
void FeedbackNexus::SetFeedback(Tagged<FeedbackType> feedback,
WriteBarrierMode mode,
Tagged<FeedbackExtraType> feedback_extra,
WriteBarrierMode mode_extra) {
config()->SetFeedbackPair(vector(), slot(), feedback, mode, feedback_extra,
mode_extra);
}
template <typename F>
void FeedbackNexus::IterateMapsWithUnclearedHandler(F function) const {
for (FeedbackIterator it(this); !it.done(); it.Advance()) {
DirectHandle<Map> map = config()->NewHandle(it.map());
if (!it.handler().IsCleared()) {
function(map);
}
}
}
Builtin FeedbackNexus::GetLoadICHandlerForFieldIndex(int field_index,
bool is_inobject,
bool is_double) {
if (is_double) return Builtin::kLoadICDoubleFieldBaseline;
if (is_inobject) {
int in_object_index = field_index - JSObject::kHeaderSize / kTaggedSize;
DCHECK_GE(in_object_index, 0);
int kMaxIndex = 7;
if (in_object_index > kMaxIndex)
return Builtin::kLoadICInObjectNonDoubleFieldBaseline;
LOAD_IC_IN_OBJECT_FIELD_WITH_INDEX_HANDLER_LIST(
, ASSERT_BUILTIN_ID_CONSECUTIVE)
int32_t builtin_id =
static_cast<int32_t>(Builtin::kLoadICInObjectNonDoubleFieldBaseline) +
in_object_index + 1;
return static_cast<Builtin>(builtin_id);
} else {
int out_of_object_index =
field_index - OFFSET_OF_DATA_START(FixedArray) / kTaggedSize;
DCHECK_GE(out_of_object_index, 0);
int kMaxIndex = 3;
if (out_of_object_index > kMaxIndex)
return Builtin::kLoadICOutOfObjectNonDoubleFieldBaseline;
LOAD_IC_OUT_OF_OBJECT_FIELD_WITH_INDEX_HANDLER_LIST(
, ASSERT_BUILTIN_ID_CONSECUTIVE)
int32_t builtin_id =
static_cast<int32_t>(
Builtin::kLoadICOutOfObjectNonDoubleFieldBaseline) +
out_of_object_index + 1;
return static_cast<Builtin>(builtin_id);
}
}
#undef ASSERT_BUILTIN_ID_CONSECUTIVE
}
#include "src/objects/object-macros-undef.h"
#endif