#include "src/execution/tiering-manager.h"
#include <optional>
#include "src/base/platform/platform.h"
#include "src/baseline/baseline.h"
#include "src/codegen/assembler.h"
#include "src/codegen/compilation-cache.h"
#include "src/codegen/compiler.h"
#include "src/codegen/pending-optimization-table.h"
#include "src/common/globals.h"
#include "src/diagnostics/code-tracer.h"
#include "src/execution/execution.h"
#include "src/execution/frames-inl.h"
#include "src/flags/flags.h"
#include "src/handles/global-handles.h"
#include "src/init/bootstrapper.h"
#include "src/interpreter/interpreter.h"
#include "src/objects/code-kind.h"
#include "src/objects/code.h"
#include "src/tracing/trace-event.h"
#ifdef OHOS_JS_ENGINE
#include "../../../arkweb/chromium_ext/v8/trace.h"
#endif
#ifdef V8_ENABLE_SPARKPLUG
#include "src/baseline/baseline-batch-compiler.h"
#endif
namespace v8 {
namespace internal {
#define OPTIMIZATION_REASON_LIST(V) \
V(DoNotOptimize, "do not optimize") \
V(HotAndStable, "hot and stable")
enum class OptimizationReason : uint8_t {
#define OPTIMIZATION_REASON_CONSTANTS(Constant, message) k##Constant,
OPTIMIZATION_REASON_LIST(OPTIMIZATION_REASON_CONSTANTS)
#undef OPTIMIZATION_REASON_CONSTANTS
};
char const* OptimizationReasonToString(OptimizationReason reason) {
static char const* reasons[] = {
#define OPTIMIZATION_REASON_TEXTS(Constant, message) message,
OPTIMIZATION_REASON_LIST(OPTIMIZATION_REASON_TEXTS)
#undef OPTIMIZATION_REASON_TEXTS
};
size_t const index = static_cast<size_t>(reason);
DCHECK_LT(index, arraysize(reasons));
return reasons[index];
}
#undef OPTIMIZATION_REASON_LIST
std::ostream& operator<<(std::ostream& os, OptimizationReason reason) {
return os << OptimizationReasonToString(reason);
}
class OptimizationDecision {
public:
static constexpr OptimizationDecision Maglev() {
return {OptimizationReason::kHotAndStable, CodeKind::MAGLEV,
ConcurrencyMode::kConcurrent};
}
static constexpr OptimizationDecision TurbofanHotAndStable() {
return {OptimizationReason::kHotAndStable, CodeKind::TURBOFAN_JS,
ConcurrencyMode::kConcurrent};
}
static constexpr OptimizationDecision DoNotOptimize() {
return {OptimizationReason::kDoNotOptimize,
CodeKind::TURBOFAN_JS, ConcurrencyMode::kConcurrent};
}
constexpr bool should_optimize() const {
return optimization_reason != OptimizationReason::kDoNotOptimize;
}
OptimizationReason optimization_reason;
CodeKind code_kind;
ConcurrencyMode concurrency_mode;
private:
OptimizationDecision() = default;
constexpr OptimizationDecision(OptimizationReason optimization_reason,
CodeKind code_kind,
ConcurrencyMode concurrency_mode)
: optimization_reason(optimization_reason),
code_kind(code_kind),
concurrency_mode(concurrency_mode) {}
};
static_assert(sizeof(OptimizationDecision) <= kInt32Size);
namespace {
void TraceInOptimizationQueue(Tagged<JSFunction> function,
CodeKind current_code_kind) {
if (v8_flags.trace_opt_verbose) {
PrintF("[not marking function %s (%s) for optimization: already queued]\n",
function->DebugNameCStr().get(),
CodeKindToString(current_code_kind));
}
}
void TraceHeuristicOptimizationDisallowed(Tagged<JSFunction> function) {
if (v8_flags.trace_opt_verbose) {
PrintF(
"[not marking function %s for optimization: marked with "
"%%PrepareFunctionForOptimization for manual optimization]\n",
function->DebugNameCStr().get());
}
}
void TraceRecompile(Isolate* isolate, Tagged<JSFunction> function,
OptimizationDecision d) {
if (v8_flags.trace_opt) {
CodeTracer::Scope scope(isolate->GetCodeTracer());
PrintF(scope.file(), "[marking ");
ShortPrint(function, scope.file());
PrintF(scope.file(), " for optimization to %s, %s, reason: %s",
CodeKindToString(d.code_kind), ToString(d.concurrency_mode),
OptimizationReasonToString(d.optimization_reason));
PrintF(scope.file(), "]\n");
}
}
}
void TraceManualRecompile(Tagged<JSFunction> function, CodeKind code_kind,
ConcurrencyMode concurrency_mode) {
if (v8_flags.trace_opt) {
PrintF("[manually marking ");
ShortPrint(function);
PrintF(" for optimization to %s, %s]\n", CodeKindToString(code_kind),
ToString(concurrency_mode));
}
}
void TieringManager::Optimize(Tagged<JSFunction> function,
OptimizationDecision d) {
DCHECK(d.should_optimize());
TraceRecompile(isolate_, function, d);
function->RequestOptimization(isolate_, d.code_kind, d.concurrency_mode);
}
void TieringManager::MarkForTurboFanOptimization(Tagged<JSFunction> function) {
Optimize(function, OptimizationDecision::TurbofanHotAndStable());
}
namespace {
bool FirstTimeTierUpToSparkplug(Isolate* isolate, Tagged<JSFunction> function) {
return !function->has_feedback_vector() ||
(function->ActiveTierIsIgnition(isolate) &&
CanCompileWithBaseline(isolate, function->shared()) &&
function->shared()->cached_tiering_decision() ==
CachedTieringDecision::kPending);
}
bool TiersUpToMaglev(CodeKind code_kind) {
return V8_LIKELY(maglev::IsMaglevEnabled()) &&
CodeKindIsUnoptimizedJSFunction(code_kind);
}
bool TiersUpToMaglev(std::optional<CodeKind> code_kind) {
return code_kind.has_value() && TiersUpToMaglev(code_kind.value());
}
int InterruptBudgetFor(Isolate* isolate, std::optional<CodeKind> code_kind,
Tagged<JSFunction> function,
CachedTieringDecision cached_tiering_decision,
int bytecode_length) {
if (function->tiering_in_progress()) return INT_MAX / 2;
const std::optional<CodeKind> existing_request =
function->GetRequestedOptimizationIfAny(isolate);
if (existing_request == CodeKind::TURBOFAN_JS ||
(code_kind.has_value() && code_kind.value() == CodeKind::TURBOFAN_JS)) {
return v8_flags.invocation_count_for_osr * bytecode_length;
}
if (maglev::IsMaglevOsrEnabled() && existing_request == CodeKind::MAGLEV) {
return v8_flags.invocation_count_for_maglev_osr * bytecode_length;
}
if (TiersUpToMaglev(code_kind) &&
!function->IsTieringRequestedOrInProgress()) {
if (v8_flags.profile_guided_optimization) {
switch (cached_tiering_decision) {
case CachedTieringDecision::kDelayMaglev:
return (std::max(v8_flags.invocation_count_for_maglev,
v8_flags.minimum_invocations_after_ic_update) +
v8_flags.invocation_count_for_maglev_with_delay) *
bytecode_length;
case CachedTieringDecision::kEarlyMaglev:
case CachedTieringDecision::kEarlyTurbofan:
return v8_flags.invocation_count_for_early_optimization *
bytecode_length;
case CachedTieringDecision::kPending:
case CachedTieringDecision::kEarlySparkplug:
case CachedTieringDecision::kNormal:
return v8_flags.invocation_count_for_maglev * bytecode_length;
}
SBXCHECK(false);
}
return v8_flags.invocation_count_for_maglev * bytecode_length;
}
return v8_flags.invocation_count_for_turbofan * bytecode_length;
}
}
int TieringManager::InterruptBudgetFor(
Isolate* isolate, Tagged<JSFunction> function,
std::optional<CodeKind> override_active_tier) {
DCHECK(function->shared()->is_compiled());
const int bytecode_length =
function->shared()->GetBytecodeArray(isolate)->length();
if (FirstTimeTierUpToSparkplug(isolate, function)) {
return bytecode_length * v8_flags.invocation_count_for_feedback_allocation;
}
DCHECK(function->has_feedback_vector());
if (bytecode_length > v8_flags.max_optimized_bytecode_size) {
return INT_MAX / 2;
}
return ::i::InterruptBudgetFor(
isolate,
override_active_tier ? override_active_tier
: function->GetActiveTier(isolate),
function, function->shared()->cached_tiering_decision(), bytecode_length);
}
namespace {
void TrySetOsrUrgency(Isolate* isolate, Tagged<JSFunction> function,
int osr_urgency) {
Tagged<SharedFunctionInfo> shared = function->shared();
if (V8_UNLIKELY(!v8_flags.use_osr)) return;
if (V8_UNLIKELY(shared->optimization_disabled(
(!v8_flags.maglev ||
(v8_flags.turbofan && function->ActiveTierIsMaglev(isolate)))
? CodeKind::TURBOFAN_JS
: CodeKind::MAGLEV))) {
return;
}
Tagged<FeedbackVector> fv = function->feedback_vector();
if (V8_UNLIKELY(v8_flags.trace_osr)) {
CodeTracer::Scope scope(isolate->GetCodeTracer());
PrintF(scope.file(),
"[OSR - setting osr urgency. function: %s, old urgency: %d, new "
"urgency: %d]\n",
function->DebugNameCStr().get(), fv->osr_urgency(), osr_urgency);
}
DCHECK_GE(osr_urgency, fv->osr_urgency());
fv->set_osr_urgency(osr_urgency);
}
void TryIncrementOsrUrgency(Isolate* isolate, Tagged<JSFunction> function) {
int old_urgency = function->feedback_vector()->osr_urgency();
int new_urgency = std::min(old_urgency + 1, FeedbackVector::kMaxOsrUrgency);
TrySetOsrUrgency(isolate, function, new_urgency);
}
void TryRequestOsrAtNextOpportunity(Isolate* isolate,
Tagged<JSFunction> function) {
TrySetOsrUrgency(isolate, function, FeedbackVector::kMaxOsrUrgency);
}
}
void TieringManager::RequestOsrAtNextOpportunity(Tagged<JSFunction> function) {
DisallowGarbageCollection no_gc;
TryRequestOsrAtNextOpportunity(isolate_, function);
}
void TieringManager::MaybeOptimizeFrame(Tagged<JSFunction> function,
CodeKind current_code_kind) {
const bool tiering_in_progress = function->tiering_in_progress();
const bool osr_in_progress =
function->feedback_vector()->osr_tiering_in_progress();
static_assert(kTieringStateInProgressBlocksTierup);
if (V8_UNLIKELY(tiering_in_progress) || V8_UNLIKELY(osr_in_progress)) {
if (v8_flags.concurrent_recompilation_front_running &&
((tiering_in_progress && function->ActiveTierIsMaglev(isolate_)) ||
(osr_in_progress &&
function->feedback_vector()->maybe_has_optimized_osr_code()))) {
isolate_->IncreaseConcurrentOptimizationPriority(CodeKind::TURBOFAN_JS,
function->shared());
}
TraceInOptimizationQueue(function, current_code_kind);
return;
}
if (V8_UNLIKELY(v8_flags.allow_natives_syntax) &&
ManualOptimizationTable::IsMarkedForManualOptimization(isolate_,
function)) {
TraceHeuristicOptimizationDisallowed(function);
return;
}
if (V8_UNLIKELY(function->shared()->optimization_disabled(
current_code_kind == CodeKind::MAGLEV ? CodeKind::TURBOFAN_JS
: CodeKind::MAGLEV))) {
return;
}
if (V8_UNLIKELY(v8_flags.always_osr)) {
TryRequestOsrAtNextOpportunity(isolate_, function);
}
const bool maglev_osr = maglev::IsMaglevOsrEnabled();
const CodeKinds available_kinds = function->GetAvailableCodeKinds(isolate_);
const bool waiting_for_tierup =
(current_code_kind < CodeKind::TURBOFAN_JS &&
(available_kinds & CodeKindFlag::TURBOFAN_JS)) ||
(maglev_osr && current_code_kind < CodeKind::MAGLEV &&
(available_kinds & CodeKindFlag::MAGLEV));
if (function->IsOptimizationRequested(isolate_) || waiting_for_tierup) {
if (V8_UNLIKELY(maglev_osr && current_code_kind == CodeKind::MAGLEV &&
(!v8_flags.osr_from_maglev ||
isolate_->EfficiencyModeEnabled() ||
isolate_->BatterySaverModeEnabled()))) {
return;
}
TryIncrementOsrUrgency(isolate_, function);
return;
}
const std::optional<CodeKind> existing_request =
function->GetRequestedOptimizationIfAny(isolate_);
DCHECK(existing_request != CodeKind::TURBOFAN_JS);
DCHECK(!function->HasAvailableCodeKind(isolate_, CodeKind::TURBOFAN_JS));
OptimizationDecision d =
ShouldOptimize(function->feedback_vector(), current_code_kind);
if (V8_UNLIKELY(!isolate_->EfficiencyModeEnabled() && !maglev_osr &&
d.should_optimize() && d.code_kind == CodeKind::MAGLEV)) {
bool is_marked_for_maglev_optimization =
existing_request == CodeKind::MAGLEV ||
(available_kinds & CodeKindFlag::MAGLEV);
if (is_marked_for_maglev_optimization) {
d = ShouldOptimize(function->feedback_vector(), CodeKind::MAGLEV);
}
}
if (V8_UNLIKELY(isolate_->EfficiencyModeEnabled() &&
d.code_kind != CodeKind::TURBOFAN_JS)) {
d.concurrency_mode = ConcurrencyMode::kSynchronous;
}
if (d.should_optimize()) Optimize(function, d);
}
OptimizationDecision TieringManager::ShouldOptimize(
Tagged<FeedbackVector> feedback_vector, CodeKind current_code_kind) {
Tagged<SharedFunctionInfo> shared = feedback_vector->shared_function_info();
if (current_code_kind == CodeKind::TURBOFAN_JS) {
return OptimizationDecision::DoNotOptimize();
}
if (TiersUpToMaglev(current_code_kind) &&
shared->PassesFilter(v8_flags.maglev_filter) &&
!shared->maglev_compilation_failed()) {
if (v8_flags.profile_guided_optimization &&
shared->cached_tiering_decision() ==
CachedTieringDecision::kEarlyTurbofan) {
return OptimizationDecision::TurbofanHotAndStable();
}
return OptimizationDecision::Maglev();
}
if (V8_UNLIKELY(!v8_flags.turbofan ||
!shared->PassesFilter(v8_flags.turbo_filter) ||
(v8_flags.efficiency_mode_disable_turbofan &&
isolate_->EfficiencyModeEnabled()) ||
isolate_->BatterySaverModeEnabled())) {
return OptimizationDecision::DoNotOptimize();
}
if (isolate_->EfficiencyModeEnabled() &&
v8_flags.efficiency_mode_delay_turbofan_multiply &&
feedback_vector->invocation_count() <
v8_flags.invocation_count_for_turbofan *
v8_flags.efficiency_mode_delay_turbofan_multiply) {
return OptimizationDecision::DoNotOptimize();
}
Tagged<BytecodeArray> bytecode = shared->GetBytecodeArray(isolate_);
if (bytecode->length() > v8_flags.max_optimized_bytecode_size) {
return OptimizationDecision::DoNotOptimize();
}
return OptimizationDecision::TurbofanHotAndStable();
}
namespace {
bool ShouldResetInterruptBudgetByICChange(
CachedTieringDecision cached_tiering_decision) {
switch (cached_tiering_decision) {
case CachedTieringDecision::kEarlyMaglev:
case CachedTieringDecision::kEarlyTurbofan:
return false;
case CachedTieringDecision::kPending:
case CachedTieringDecision::kEarlySparkplug:
case CachedTieringDecision::kDelayMaglev:
case CachedTieringDecision::kNormal:
return true;
}
SBXCHECK(false);
}
}
void TieringManager::NotifyICChanged(Tagged<FeedbackVector> vector) {
CodeKind code_kind = vector->shared_function_info()->HasBaselineCode()
? CodeKind::BASELINE
: CodeKind::INTERPRETED_FUNCTION;
if (code_kind == CodeKind::INTERPRETED_FUNCTION &&
CanCompileWithBaseline(isolate_, vector->shared_function_info()) &&
vector->shared_function_info()->cached_tiering_decision() ==
CachedTieringDecision::kPending) {
return;
}
OptimizationDecision decision = ShouldOptimize(vector, code_kind);
if (decision.should_optimize()) {
Tagged<SharedFunctionInfo> shared = vector->shared_function_info();
int bytecode_length = shared->GetBytecodeArray(isolate_)->length();
Tagged<FeedbackCell> cell = vector->parent_feedback_cell();
int invocations = v8_flags.minimum_invocations_after_ic_update;
int bytecodes = std::min(bytecode_length, (kMaxInt >> 1) / invocations);
int new_budget = invocations * bytecodes;
int current_budget = cell->interrupt_budget();
if (v8_flags.profile_guided_optimization &&
shared->cached_tiering_decision() <=
CachedTieringDecision::kEarlySparkplug) {
DCHECK_LT(v8_flags.invocation_count_for_early_optimization,
FeedbackVector::kInvocationCountBeforeStableDeoptSentinel);
if (vector->invocation_count_before_stable() <
v8_flags.invocation_count_for_early_optimization) {
int new_invocation_count_before_stable;
if (vector->interrupt_budget_reset_by_ic_change()) {
int new_consumed_budget = new_budget - current_budget;
new_invocation_count_before_stable =
vector->invocation_count_before_stable(kRelaxedLoad) +
std::ceil(static_cast<float>(new_consumed_budget) / bytecodes);
} else {
int total_consumed_budget =
(maglev::IsMaglevEnabled()
? v8_flags.invocation_count_for_maglev
: v8_flags.invocation_count_for_turbofan) *
bytecodes -
current_budget;
new_invocation_count_before_stable =
std::ceil(static_cast<float>(total_consumed_budget) / bytecodes);
}
if (new_invocation_count_before_stable >=
v8_flags.invocation_count_for_early_optimization) {
vector->set_invocation_count_before_stable(
v8_flags.invocation_count_for_early_optimization, kRelaxedStore);
shared->set_cached_tiering_decision(CachedTieringDecision::kNormal);
} else {
vector->set_invocation_count_before_stable(
new_invocation_count_before_stable, kRelaxedStore);
}
} else {
shared->set_cached_tiering_decision(CachedTieringDecision::kNormal);
}
}
if (!v8_flags.profile_guided_optimization ||
ShouldResetInterruptBudgetByICChange(
shared->cached_tiering_decision())) {
if (new_budget > current_budget) {
if (v8_flags.trace_opt_verbose) {
PrintF("[delaying optimization of %s, IC changed]\n",
shared->DebugNameCStr().get());
}
vector->set_interrupt_budget_reset_by_ic_change(true);
cell->set_interrupt_budget(new_budget);
}
}
}
}
TieringManager::OnInterruptTickScope::OnInterruptTickScope() {
TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
"V8.MarkCandidatesForOptimization");
#ifdef OHOS_JS_ENGINE
auto trace = HiTrace("RCS_v8.compile_V8.MarkCandidatesForOptimization");
#endif
}
void TieringManager::OnInterruptTick(DirectHandle<JSFunction> function,
CodeKind code_kind) {
IsCompiledScope is_compiled_scope(
function->shared()->is_compiled_scope(isolate_));
const bool had_feedback_vector = function->has_feedback_vector();
const bool first_time_tiered_up_to_sparkplug =
FirstTimeTierUpToSparkplug(isolate_, *function);
const bool maybe_had_optimized_osr_code =
had_feedback_vector &&
function->feedback_vector()->maybe_has_optimized_osr_code();
const bool compile_sparkplug =
CanCompileWithBaseline(isolate_, function->shared()) &&
function->ActiveTierIsIgnition(isolate_) && !maybe_had_optimized_osr_code;
if (!had_feedback_vector) {
if (compile_sparkplug && function->shared()->cached_tiering_decision() ==
CachedTieringDecision::kPending) {
function->shared()->set_cached_tiering_decision(
CachedTieringDecision::kEarlySparkplug);
}
JSFunction::CreateAndAttachFeedbackVector(isolate_, function,
&is_compiled_scope);
DCHECK(is_compiled_scope.is_compiled());
function->feedback_vector()->set_invocation_count(1, kRelaxedStore);
}
DCHECK(function->has_feedback_vector());
DCHECK(function->shared()->is_compiled());
DCHECK(function->shared()->HasBytecodeArray());
if (compile_sparkplug) {
#ifdef V8_ENABLE_SPARKPLUG
if (v8_flags.baseline_batch_compilation) {
isolate_->baseline_batch_compiler()->EnqueueFunction(function);
} else {
IsCompiledScope inner_is_compiled_scope(
function->shared()->is_compiled_scope(isolate_));
Compiler::CompileBaseline(isolate_, function, Compiler::CLEAR_EXCEPTION,
&inner_is_compiled_scope);
}
#else
UNREACHABLE();
#endif
}
if (first_time_tiered_up_to_sparkplug) {
if (had_feedback_vector) {
if (function->shared()->cached_tiering_decision() ==
CachedTieringDecision::kPending) {
function->shared()->set_cached_tiering_decision(
CachedTieringDecision::kEarlySparkplug);
}
function->SetInterruptBudget(isolate_, BudgetModification::kRaise);
}
return;
}
if (V8_UNLIKELY(!isolate_->use_optimizer())) {
function->SetInterruptBudget(isolate_, BudgetModification::kRaise);
return;
}
DisallowGarbageCollection no_gc;
OnInterruptTickScope scope;
Tagged<JSFunction> function_obj = *function;
MaybeOptimizeFrame(function_obj, code_kind);
DCHECK(had_feedback_vector);
function->SetInterruptBudget(isolate_, BudgetModification::kRaise);
}
}
}