* Copyright (c) 2022-2024 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ecmascript/compiler/assembler/aarch64/assembler_aarch64.h"
#include "ecmascript/compiler/trampoline/aarch64/common_call.h"
#include "ecmascript/js_generator_object.h"
#include "ecmascript/message_string.h"
namespace panda::ecmascript::aarch64 {
using Label = panda::ecmascript::Label;
#define __ assembler->
void AsmInterpreterCall::AsmInterpreterEntry(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(AsmInterpreterEntry));
Label target;
size_t begin = __ GetCurrentPosition();
PushAsmInterpEntryFrame(assembler);
__ Bl(&target);
PopAsmInterpEntryFrame(assembler);
size_t end = __ GetCurrentPosition();
if ((end - begin) != FrameCompletionPos::ARM64EntryFrameDuration) {
LOG_COMPILER(FATAL) << (end - begin) << " != " << FrameCompletionPos::ARM64EntryFrameDuration
<< "This frame has been modified, and the offset EntryFrameDuration should be updated too.";
}
__ Ret();
__ Bind(&target);
{
AsmInterpEntryDispatch(assembler);
}
}
void AsmInterpreterCall::AsmInterpEntryDispatch(ExtendedAssembler *assembler)
{
Label notJSFunction;
Label callNativeEntry;
Label callJSFunctionEntry;
Label notCallable;
Register glueRegister = x0;
Register argcRegister = w4;
Register argvRegister = x5;
Register callTargetRegister = x1;
Register callFieldRegister = x3;
Register bitFieldRegister = x16;
Register tempRegister = x17;
Register functionTypeRegister = w18;
__ Ldr(tempRegister, MemoryOperand(callTargetRegister, TaggedObject::HCLASS_OFFSET));
__ And(tempRegister, tempRegister, LogicalImmediate::Create(TaggedObject::GC_STATE_MASK, X_REG_SIZE));
__ Ldr(bitFieldRegister, MemoryOperand(tempRegister, JSHClass::BIT_FIELD_OFFSET));
__ And(functionTypeRegister, bitFieldRegister.W(), LogicalImmediate::Create(0xFF, W_REG_SIZE));
__ Mov(tempRegister.W(), Immediate(static_cast<int64_t>(JSType::JS_FUNCTION_FIRST)));
__ Cmp(functionTypeRegister, tempRegister.W());
__ B(Condition::LO, ¬JSFunction);
__ Mov(tempRegister.W(), Immediate(static_cast<int64_t>(JSType::JS_FUNCTION_LAST)));
__ Cmp(functionTypeRegister, tempRegister.W());
__ B(Condition::LS, &callJSFunctionEntry);
__ Bind(¬JSFunction);
{
__ Tst(bitFieldRegister,
LogicalImmediate::Create(static_cast<int64_t>(1ULL << JSHClass::CallableBit::START_BIT), X_REG_SIZE));
__ B(Condition::EQ, ¬Callable);
CallNativeEntry(assembler, false);
}
__ Bind(&callNativeEntry);
CallNativeEntry(assembler, true);
__ Bind(&callJSFunctionEntry);
{
__ Tbnz(callFieldRegister, Method::IsNativeBit::START_BIT, &callNativeEntry);
__ Add(argvRegister, argvRegister, Immediate(NUM_MANDATORY_JSFUNC_ARGS * JSTaggedValue::TaggedTypeSize()));
JSCallCommonEntry(assembler, JSCallMode::CALL_ENTRY, FrameTransitionType::OTHER_TO_BASELINE_CHECK);
}
__ Bind(¬Callable);
{
Register runtimeId = x11;
Register trampoline = x12;
__ Mov(runtimeId, Immediate(kungfu::RuntimeStubCSigns::ID_ThrowNotCallableException));
__ Add(trampoline, glueRegister, Operand(runtimeId, LSL, 3));
__ Ldr(trampoline, MemoryOperand(trampoline, JSThread::GlueData::GetRTStubEntriesOffset(false)));
#ifdef ENABLE_CMC_IR_FIX_REGISTER
__ Mov(x28, glueRegister);
#endif
__ Blr(trampoline);
__ UpdateGlueAndReadBarrier();
__ Ret();
}
}
void AsmInterpreterCall::JSCallCommonEntry(ExtendedAssembler *assembler,
JSCallMode mode, FrameTransitionType type)
{
Label stackOverflow;
Register glueRegister = __ GlueRegister();
__ UpdateGlueAndReadBarrier(glueRegister);
Register fpRegister = __ AvailableRegister1();
Register currentSlotRegister = __ AvailableRegister3();
Register callFieldRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
Register argcRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARGC);
if (!kungfu::AssemblerModule::IsJumpToCallCommonEntry(mode) || type == FrameTransitionType::BASELINE_TO_OTHER ||
type == FrameTransitionType::BASELINE_TO_BASELINE_CHECK) {
__ PushFpAndLr();
}
__ Mov(fpRegister, sp);
__ Mov(currentSlotRegister, sp);
{
[[maybe_unused]] TempRegister1Scope scope(assembler);
Register tempRegister = __ TempRegister1();
__ Ldr(tempRegister, MemoryOperand(glueRegister, JSThread::GlueData::GetStackLimitOffset(false)));
__ Mov(sp, tempRegister);
}
Register declaredNumArgsRegister = __ AvailableRegister2();
GetDeclaredNumArgsFromCallField(assembler, callFieldRegister, declaredNumArgsRegister);
Label slowPathEntry;
Label fastPathEntry;
Label pushCallThis;
auto argc = kungfu::AssemblerModule::GetArgcFromJSCallMode(mode);
if (argc >= 0) {
__ Cmp(declaredNumArgsRegister, Immediate(argc));
} else {
__ Cmp(declaredNumArgsRegister, argcRegister);
}
__ B(Condition::NE, &slowPathEntry);
__ Bind(&fastPathEntry);
JSCallCommonFastPath(assembler, mode, &pushCallThis, &stackOverflow);
__ Bind(&pushCallThis);
PushCallThis(assembler, mode, &stackOverflow, type);
__ Bind(&slowPathEntry);
JSCallCommonSlowPath(assembler, mode, &fastPathEntry, &pushCallThis, &stackOverflow);
__ Bind(&stackOverflow);
if (kungfu::AssemblerModule::IsJumpToCallCommonEntry(mode)) {
__ Mov(sp, fpRegister);
[[maybe_unused]] TempRegister1Scope scope(assembler);
Register temp = __ TempRegister1();
if (glueRegister != x19) {
__ Mov(x19, glueRegister);
}
Register acc = x23;
__ Mov(acc, Immediate(JSTaggedValue::VALUE_EXCEPTION));
Register methodRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::METHOD);
Register callTargetRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_TARGET);
__ Mov(temp, callTargetRegister);
__ Ldr(x20, MemoryOperand(methodRegister, Method::NATIVE_POINTER_OR_BYTECODE_ARRAY_OFFSET));
__ Ldr(x22, MemoryOperand(temp, JSFunction::RAW_PROFILE_TYPE_INFO_OFFSET));
__ Ldr(x22, MemoryOperand(x22, ProfileTypeInfoCell::VALUE_OFFSET));
__ Ldr(x21, MemoryOperand(methodRegister, Method::CONSTANT_POOL_OFFSET));
__ Mov(temp, Immediate(kungfu::BytecodeStubCSigns::ID_ThrowStackOverflowException));
__ Add(temp, glueRegister, Operand(temp, UXTW, 3));
__ Ldr(temp, MemoryOperand(temp, JSThread::GlueData::GetBCStubEntriesOffset(false)));
__ Br(temp);
} else {
[[maybe_unused]] TempRegister1Scope scope(assembler);
Register temp = __ TempRegister1();
ThrowStackOverflowExceptionAndReturn(assembler, glueRegister, fpRegister, temp);
}
}
void AsmInterpreterCall::JSCallCommonFastPath(ExtendedAssembler *assembler, JSCallMode mode, Label *pushCallThis,
Label *stackOverflow)
{
Register glueRegister = __ GlueRegister();
auto argc = kungfu::AssemblerModule::GetArgcFromJSCallMode(mode);
Register currentSlotRegister = __ AvailableRegister3();
if (argc < 0) {
Register numRegister = __ AvailableRegister2();
Register argcRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARGC);
Register argvRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARGV);
__ Mov(numRegister, argcRegister);
[[maybe_unused]] TempRegister1Scope scope(assembler);
Register opRegister = __ TempRegister1();
PushArgsWithArgv(assembler, glueRegister, numRegister, argvRegister, opRegister,
currentSlotRegister, pushCallThis, stackOverflow);
} else {
if (argc > 2) {
Register arg2 = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG2);
__ Str(arg2, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
}
if (argc > 1) {
Register arg1 = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
__ Str(arg1, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
}
if (argc > 0) {
Register arg0 = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG0);
__ Str(arg0, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
}
if (stackOverflow != nullptr) {
[[maybe_unused]] TempRegister1Scope scope(assembler);
Register op = __ TempRegister1();
Register numRegister = __ AvailableRegister2();
__ Mov(numRegister, Immediate(argc));
StackOverflowCheck(assembler, glueRegister, currentSlotRegister, numRegister, op, stackOverflow);
}
}
}
void AsmInterpreterCall::JSCallCommonSlowPath(ExtendedAssembler *assembler, JSCallMode mode,
Label *fastPathEntry, Label *pushCallThis, Label *stackOverflow)
{
Register glueRegister = __ GlueRegister();
Register callFieldRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
Register argcRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARGC);
Register argvRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARGV);
Register currentSlotRegister = __ AvailableRegister3();
Register arg0 = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG0);
Register arg1 = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
Label noExtraEntry;
Label pushArgsEntry;
auto argc = kungfu::AssemblerModule::GetArgcFromJSCallMode(mode);
Register declaredNumArgsRegister = __ AvailableRegister2();
__ Tbz(callFieldRegister, Method::HaveExtraBit::START_BIT, &noExtraEntry);
{
[[maybe_unused]] TempRegister1Scope scope1(assembler);
Register tempArgcRegister = __ TempRegister1();
if (argc >= 0) {
__ PushArgc(argc, tempArgcRegister, currentSlotRegister);
} else {
__ PushArgc(argcRegister, tempArgcRegister, currentSlotRegister);
}
}
__ Bind(&noExtraEntry);
{
if (argc == 0) {
{
[[maybe_unused]] TempRegister1Scope scope(assembler);
Register tempRegister = __ TempRegister1();
PushUndefinedWithArgc(assembler, glueRegister, declaredNumArgsRegister, tempRegister,
currentSlotRegister, nullptr, stackOverflow);
}
__ B(fastPathEntry);
return;
}
[[maybe_unused]] TempRegister1Scope scope1(assembler);
Register diffRegister = __ TempRegister1();
if (argc >= 0) {
__ Sub(diffRegister.W(), declaredNumArgsRegister.W(), Immediate(argc));
} else {
__ Sub(diffRegister.W(), declaredNumArgsRegister.W(), argcRegister.W());
}
[[maybe_unused]] TempRegister2Scope scope2(assembler);
Register tempRegister = __ TempRegister2();
PushUndefinedWithArgc(assembler, glueRegister, diffRegister, tempRegister,
currentSlotRegister, &pushArgsEntry, stackOverflow);
__ B(fastPathEntry);
}
__ Bind(&pushArgsEntry);
{
__ Tbnz(callFieldRegister, Method::HaveExtraBit::START_BIT, fastPathEntry);
if (argc == 1) {
__ B(pushCallThis);
return;
}
__ Cmp(declaredNumArgsRegister, Immediate(0));
__ B(Condition::EQ, pushCallThis);
if (argc < 0) {
[[maybe_unused]] TempRegister1Scope scope(assembler);
Register opRegister = __ TempRegister1();
PushArgsWithArgv(assembler, glueRegister, declaredNumArgsRegister,
argvRegister, opRegister,
currentSlotRegister, nullptr, stackOverflow);
} else if (argc > 0) {
Label pushArgs0;
if (argc > 2) {
__ Cmp(declaredNumArgsRegister, Immediate(1));
__ B(Condition::EQ, &pushArgs0);
__ Str(arg1, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
}
if (argc > 1) {
__ Bind(&pushArgs0);
__ Str(arg0, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
}
}
__ B(pushCallThis);
}
}
Register AsmInterpreterCall::GetThisRegsiter(ExtendedAssembler *assembler, JSCallMode mode, Register defaultRegister)
{
switch (mode) {
case JSCallMode::CALL_GETTER:
case JSCallMode::CALL_THIS_ARG0:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG0);
case JSCallMode::CALL_SETTER:
case JSCallMode::CALL_THIS_ARG1:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
case JSCallMode::CALL_THIS_ARG2:
case JSCallMode::CALL_THIS_ARG2_WITH_RETURN:
case JSCallMode::CALL_CONSTRUCTOR_WITH_ARGV:
case JSCallMode::SUPER_CALL_WITH_ARGV:
case JSCallMode::SUPER_CALL_SPREAD_WITH_ARGV:
case JSCallMode::CALL_THIS_WITH_ARGV:
case JSCallMode::CALL_THIS_ARGV_WITH_RETURN:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG2);
case JSCallMode::CALL_THIS_ARG3:
case JSCallMode::CALL_THIS_ARG3_WITH_RETURN:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG3);
case JSCallMode::CALL_FROM_AOT:
case JSCallMode::CALL_ENTRY: {
Register argvRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
__ Ldur(defaultRegister, MemoryOperand(argvRegister, -FRAME_SLOT_SIZE));
return defaultRegister;
}
default:
LOG_ECMA(FATAL) << "this branch is unreachable";
UNREACHABLE();
}
return invalidReg;
}
Register AsmInterpreterCall::GetNewTargetRegsiter(ExtendedAssembler *assembler, JSCallMode mode,
Register defaultRegister)
{
switch (mode) {
case JSCallMode::CALL_CONSTRUCTOR_WITH_ARGV:
case JSCallMode::CALL_THIS_WITH_ARGV:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_TARGET);
case JSCallMode::SUPER_CALL_WITH_ARGV:
case JSCallMode::SUPER_CALL_SPREAD_WITH_ARGV:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG3);
case JSCallMode::CALL_FROM_AOT:
case JSCallMode::CALL_ENTRY: {
Register argvRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
__ Ldur(defaultRegister, MemoryOperand(argvRegister, -2 * FRAME_SLOT_SIZE));
return defaultRegister;
}
default:
LOG_ECMA(FATAL) << "this branch is unreachable";
UNREACHABLE();
}
return invalidReg;
}
void AsmInterpreterCall::PushCallThisRangeAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallThisRangeAndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_WITH_ARGV, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallRangeAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallRangeAndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_WITH_ARGV, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallNewAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallNewAndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_CONSTRUCTOR_WITH_ARGV, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushSuperCallAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushSuperCallAndDispatch));
JSCallCommonEntry(assembler, JSCallMode::SUPER_CALL_WITH_ARGV, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallArgs3AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallArgs3AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_ARG3, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallArgs2AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallArgs2AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_ARG2, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallArg1AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallArg1AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_ARG1, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallArg0AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallArg0AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_ARG0, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallThisArg0AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallThisArg0AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG0, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallThisArg1AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallThisArg1AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG1, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallThisArgs2AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallThisArgs2AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG2, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallThisArgs3AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallThisArgs3AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG3, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallRangeAndDispatchNative(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallRangeAndDispatchNative));
CallNativeWithArgv(assembler, false);
}
void AsmInterpreterCall::PushCallNewAndDispatchNative(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallNewAndDispatchNative));
CallNativeWithArgv(assembler, true);
}
void AsmInterpreterCall::PushNewTargetAndDispatchNative(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushNewTargetAndDispatchNative));
CallNativeWithArgv(assembler, true, true);
}
void AsmInterpreterCall::CallNativeWithArgv(ExtendedAssembler *assembler, bool callNew, bool hasNewTarget)
{
Register glue = x0;
Register nativeCode = x1;
Register callTarget = x2;
Register thisObj = x3;
Register argc = x4;
Register argv = x5;
Register newTarget = x6;
Register opArgc = x8;
Register opArgv = x9;
Register temp = x10;
Register currentSlotRegister = x11;
Label pushThis;
Label stackOverflow;
bool isFrameComplete = PushBuiltinFrame(assembler, glue, FrameType::BUILTIN_FRAME_WITH_ARGV, temp, argc);
__ Mov(currentSlotRegister, sp);
__ Ldr(temp, MemoryOperand(glue, JSThread::GlueData::GetStackLimitOffset(false)));
__ Mov(sp, temp);
__ Mov(opArgc, argc);
__ Mov(opArgv, argv);
PushArgsWithArgv(assembler, glue, opArgc, opArgv, temp, currentSlotRegister, &pushThis, &stackOverflow);
__ Bind(&pushThis);
if (callNew) {
if (hasNewTarget) {
__ Stp(newTarget, thisObj, MemoryOperand(currentSlotRegister, -DOUBLE_SLOT_SIZE, AddrMode::PREINDEX));
} else {
__ Stp(callTarget, thisObj, MemoryOperand(currentSlotRegister, -DOUBLE_SLOT_SIZE, AddrMode::PREINDEX));
}
} else {
__ Mov(temp, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ Stp(temp, thisObj, MemoryOperand(currentSlotRegister, -DOUBLE_SLOT_SIZE, AddrMode::PREINDEX));
}
__ Str(callTarget, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Add(temp, currentSlotRegister, Immediate(QUINTUPLE_SLOT_SIZE));
if (!isFrameComplete) {
__ Add(fp, temp, Operand(argc, LSL, 3));
}
__ Add(temp, argc, Immediate(NUM_MANDATORY_JSFUNC_ARGS));
__ Stp(glue, temp, MemoryOperand(currentSlotRegister, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
#ifdef ENABLE_CMC_IR_FIX_REGISTER
Register calleeSaveGlue = x28;
__ Mov(calleeSaveGlue, glue);
#endif
__ Add(x0, currentSlotRegister, Immediate(0));
__ Align16(currentSlotRegister);
__ Mov(sp, currentSlotRegister);
CallNativeInternal(assembler, nativeCode);
__ Ret();
__ Bind(&stackOverflow);
{
Register frameType = x11;
__ Ldr(temp, MemoryOperand(glue, JSThread::GlueData::GetLeaveFrameOffset(false)));
__ Mov(sp, temp);
__ Mov(frameType, Immediate(static_cast<int32_t>(FrameType::BUILTIN_FRAME_WITH_ARGV_STACK_OVER_FLOW_FRAME)));
__ Stp(xzr, frameType, MemoryOperand(sp, -FRAME_SLOT_SIZE * 2, AddrMode::PREINDEX));
__ Mov(temp, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ Stp(temp, temp, MemoryOperand(sp, -FRAME_SLOT_SIZE * 2, AddrMode::PREINDEX));
__ Stp(xzr, temp, MemoryOperand(sp, -FRAME_SLOT_SIZE * 2, AddrMode::PREINDEX));
__ Mov(temp, sp);
__ Add(fp, temp, Immediate(FRAME_SLOT_SIZE * 6));
Register runtimeId = x11;
Register trampoline = x12;
__ Mov(runtimeId, Immediate(kungfu::RuntimeStubCSigns::ID_ThrowStackOverflowException));
__ Add(trampoline, glue, Operand(runtimeId, LSL, 3));
__ Ldr(trampoline, MemoryOperand(trampoline, JSThread::GlueData::GetRTStubEntriesOffset(false)));
#ifdef ENABLE_CMC_IR_FIX_REGISTER
__ Mov(x28, glue);
#endif
__ Blr(trampoline);
__ UpdateGlueAndReadBarrier();
__ Mov(sp, fp);
__ RestoreFpAndLr();
__ Ret();
}
}
void AsmInterpreterCall::PushCallArgsAndDispatchNative(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallArgsAndDispatchNative));
Register nativeCode = x0;
Register glue = x1;
Register argv = x5;
Register temp = x6;
Register nativeCodeTemp = x2;
__ Mov(nativeCodeTemp, nativeCode);
__ Ldr(glue, MemoryOperand(sp, 0));
__ Add(x0, sp, Immediate(0));
PushBuiltinFrame(assembler, glue, FrameType::BUILTIN_FRAME, temp, argv);
#ifdef ENABLE_CMC_IR_FIX_REGISTER
Register calleeSaveGlue = x28;
__ Mov(calleeSaveGlue, glue);
#endif
CallNativeInternal(assembler, nativeCodeTemp);
__ Ret();
}
bool AsmInterpreterCall::PushBuiltinFrame(ExtendedAssembler *assembler, Register glue,
FrameType type, Register op, Register next)
{
__ PushFpAndLr();
__ Mov(op, sp);
__ Str(op, MemoryOperand(glue, JSThread::GlueData::GetLeaveFrameOffset(false)));
__ Mov(op, Immediate(static_cast<int32_t>(type)));
if (type == FrameType::BUILTIN_FRAME) {
__ Add(next, sp, Immediate(BuiltinFrame::GetStackArgsToFpDelta(false)));
__ Stp(next, op, MemoryOperand(sp, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Add(fp, sp, Immediate(2 * FRAME_SLOT_SIZE));
return true;
} else if (type == FrameType::BUILTIN_ENTRY_FRAME) {
__ Stp(next, op, MemoryOperand(sp, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Add(fp, sp, Immediate(2 * FRAME_SLOT_SIZE));
return true;
} else if (type == FrameType::BUILTIN_FRAME_WITH_ARGV) {
__ Stp(next, op, MemoryOperand(sp, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
return false;
} else {
LOG_ECMA(FATAL) << "this branch is unreachable";
UNREACHABLE();
}
}
void AsmInterpreterCall::CallNativeInternal(ExtendedAssembler *assembler, Register nativeCode)
{
__ Blr(nativeCode);
__ UpdateGlueAndReadBarrier();
__ Mov(sp, fp);
__ RestoreFpAndLr();
}
void AsmInterpreterCall::ResumeRspAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeRspAndDispatch));
Register glueRegister = __ GlueRegister();
Register rsp = sp;
Register currentSp = fp;
Register pc = x20;
Register jumpSizeRegister = x25;
Register ret = x23;
Register opcode = w6;
Register temp = x7;
Register bcStub = x7;
Register fpReg = x8;
int64_t fpOffset = static_cast<int64_t>(AsmInterpretedFrame::GetFpOffset(false))
- static_cast<int64_t>(AsmInterpretedFrame::GetSize(false));
int64_t spOffset = static_cast<int64_t>(AsmInterpretedFrame::GetBaseOffset(false))
- static_cast<int64_t>(AsmInterpretedFrame::GetSize(false));
int64_t thisOffset = static_cast<int64_t>(AsmInterpretedFrame::GetThisOffset(false))
- static_cast<int64_t>(AsmInterpretedFrame::GetSize(false));
ASSERT(fpOffset < 0);
ASSERT(spOffset < 0);
Label newObjectRangeReturn;
Label dispatch;
__ Ldur(fpReg, MemoryOperand(currentSp, fpOffset));
__ Cmp(jumpSizeRegister, Immediate(0));
__ B(Condition::LE, &newObjectRangeReturn);
__ Ldur(currentSp, MemoryOperand(currentSp, spOffset));
__ Add(pc, pc, Operand(jumpSizeRegister, LSL, 0));
__ Ldrb(opcode, MemoryOperand(pc, 0));
__ Bind(&dispatch);
{
__ Mov(rsp, fpReg);
__ Add(bcStub, glueRegister, Operand(opcode, UXTW, FRAME_SLOT_SIZE_LOG2));
__ Ldr(bcStub, MemoryOperand(bcStub, JSThread::GlueData::GetBCStubEntriesOffset(false)));
__ Br(bcStub);
}
Label getThis;
Label notUndefined;
__ Bind(&newObjectRangeReturn);
{
__ Cmp(ret, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ B(Condition::NE, ¬Undefined);
ASSERT(thisOffset < 0);
__ Bind(&getThis);
__ Ldur(ret, MemoryOperand(currentSp, thisOffset));
__ Ldur(currentSp, MemoryOperand(currentSp, spOffset));
__ Mov(rsp, fpReg);
__ Sub(pc, pc, jumpSizeRegister);
__ Ldrb(opcode, MemoryOperand(pc, 0));
__ Add(bcStub, glueRegister, Operand(opcode, UXTW, FRAME_SLOT_SIZE_LOG2));
__ Ldr(bcStub, MemoryOperand(bcStub, JSThread::GlueData::GetBCStubEntriesOffset(false)));
__ Br(bcStub);
}
__ Bind(¬Undefined);
{
Label notEcmaObject;
__ Mov(temp, Immediate(JSTaggedValue::TAG_HEAPOBJECT_MASK));
__ And(temp, temp, ret);
__ Cmp(temp, Immediate(0));
__ B(Condition::NE, ¬EcmaObject);
__ Ldr(temp, MemoryOperand(ret, TaggedObject::HCLASS_OFFSET));
__ And(temp, temp, LogicalImmediate::Create(TaggedObject::GC_STATE_MASK, X_REG_SIZE));
__ Ldr(temp, MemoryOperand(temp, JSHClass::BIT_FIELD_OFFSET));
__ And(temp.W(), temp.W(), LogicalImmediate::Create(0xFF, W_REG_SIZE));
__ Cmp(temp.W(), Immediate(static_cast<int64_t>(JSType::ECMA_OBJECT_LAST)));
__ B(Condition::HI, ¬EcmaObject);
__ Cmp(temp.W(), Immediate(static_cast<int64_t>(JSType::ECMA_OBJECT_FIRST)));
__ B(Condition::LO, ¬EcmaObject);
__ Ldur(currentSp, MemoryOperand(currentSp, spOffset));
__ Sub(pc, pc, jumpSizeRegister);
__ Ldrb(opcode, MemoryOperand(pc, 0));
__ B(&dispatch);
__ Bind(¬EcmaObject);
{
int64_t constructorOffset = static_cast<int64_t>(AsmInterpretedFrame::GetFunctionOffset(false))
- static_cast<int64_t>(AsmInterpretedFrame::GetSize(false));
ASSERT(constructorOffset < 0);
__ Ldur(temp, MemoryOperand(currentSp, constructorOffset));
__ Ldr(temp, MemoryOperand(temp, JSFunctionBase::METHOD_OFFSET));
__ Ldr(temp, MemoryOperand(temp, Method::EXTRA_LITERAL_INFO_OFFSET));
__ Lsr(temp.W(), temp.W(), Method::FunctionKindBits::START_BIT);
__ And(temp.W(), temp.W(),
LogicalImmediate::Create((1LU << Method::FunctionKindBits::SIZE) - 1, W_REG_SIZE));
__ Cmp(temp.W(), Immediate(static_cast<int64_t>(FunctionKind::CLASS_CONSTRUCTOR)));
__ B(Condition::LS, &getThis);
{
__ Mov(opcode, Immediate(kungfu::BytecodeStubCSigns::ID_NewObjectRangeThrowException));
__ Ldur(currentSp, MemoryOperand(currentSp, spOffset));
__ B(&dispatch);
}
}
}
}
void AsmInterpreterCall::ResumeRspAndReturn(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeRspAndReturn));
Register rsp = sp;
Register currentSp = x20;
[[maybe_unused]] TempRegister1Scope scope1(assembler);
Register fpRegister = __ TempRegister1();
int64_t offset = static_cast<int64_t>(AsmInterpretedFrame::GetFpOffset(false))
- static_cast<int64_t>(AsmInterpretedFrame::GetSize(false));
ASSERT(offset < 0);
__ Ldur(fpRegister, MemoryOperand(currentSp, offset));
__ Mov(rsp, fpRegister);
{
__ RestoreFpAndLr();
__ Mov(x0, x19);
__ Ret();
}
}
void AsmInterpreterCall::ResumeRspAndReturnBaseline(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeRspAndReturnBaseline));
Register glue = x19;
Register rsp = sp;
Register currentSp = x21;
[[maybe_unused]] TempRegister1Scope scope1(assembler);
Register fpRegister = __ TempRegister1();
int64_t fpOffset = static_cast<int64_t>(AsmInterpretedFrame::GetFpOffset(false)) -
static_cast<int64_t>(AsmInterpretedFrame::GetSize(false));
ASSERT(fpOffset < 0);
__ Ldur(fpRegister, MemoryOperand(currentSp, fpOffset));
__ Mov(rsp, fpRegister);
__ RestoreFpAndLr();
__ Mov(x0, fp);
Register ret = x0;
Register jumpSizeRegister = x22;
Label getThis;
Label notUndefined;
Label normalReturn;
Label newObjectRangeReturn;
__ Cmp(jumpSizeRegister, Immediate(0));
__ B(Condition::GT, &normalReturn);
__ Bind(&newObjectRangeReturn);
{
__ Cmp(ret, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ B(Condition::NE, ¬Undefined);
__ Bind(&getThis);
int64_t thisOffset = static_cast<int64_t>(AsmInterpretedFrame::GetThisOffset(false)) -
static_cast<int64_t>(AsmInterpretedFrame::GetSize(false));
ASSERT(thisOffset < 0);
__ Ldur(ret, MemoryOperand(currentSp, thisOffset));
__ B(&normalReturn);
__ Bind(¬Undefined);
{
Register temp = x19;
Label notEcmaObject;
__ Mov(temp, Immediate(JSTaggedValue::TAG_HEAPOBJECT_MASK));
__ And(temp, temp, ret);
__ Cmp(temp, Immediate(0));
__ B(Condition::NE, ¬EcmaObject);
__ Ldr(temp, MemoryOperand(ret, TaggedObject::HCLASS_OFFSET));
__ And(temp, temp, LogicalImmediate::Create(TaggedObject::GC_STATE_MASK, X_REG_SIZE));
__ Ldr(temp, MemoryOperand(temp, JSHClass::BIT_FIELD_OFFSET));
__ And(temp.W(), temp.W(), LogicalImmediate::Create(0xFF, W_REG_SIZE));
__ Cmp(temp.W(), Immediate(static_cast<int64_t>(JSType::ECMA_OBJECT_LAST)));
__ B(Condition::HI, ¬EcmaObject);
__ Cmp(temp.W(), Immediate(static_cast<int64_t>(JSType::ECMA_OBJECT_FIRST)));
__ B(Condition::LO, ¬EcmaObject);
__ B(&normalReturn);
__ Bind(¬EcmaObject);
{
int64_t funcOffset = static_cast<int64_t>(AsmInterpretedFrame::GetFunctionOffset(false)) -
static_cast<int64_t>(AsmInterpretedFrame::GetSize(false));
ASSERT(funcOffset < 0);
__ Ldur(temp, MemoryOperand(currentSp, funcOffset));
__ Ldr(temp, MemoryOperand(temp, JSFunctionBase::METHOD_OFFSET));
__ Ldr(temp, MemoryOperand(temp, Method::EXTRA_LITERAL_INFO_OFFSET));
__ Lsr(temp.W(), temp.W(), Method::FunctionKindBits::START_BIT);
__ And(temp.W(), temp.W(),
LogicalImmediate::Create((1LU << Method::FunctionKindBits::SIZE) - 1, W_REG_SIZE));
__ Cmp(temp.W(), Immediate(static_cast<int64_t>(FunctionKind::CLASS_CONSTRUCTOR)));
__ B(Condition::LS, &getThis);
}
}
}
__ Bind(&normalReturn);
__ Ret();
}
void AsmInterpreterCall::ResumeCaughtFrameAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeCaughtFrameAndDispatch));
Register glue = x19;
Register pc = x20;
Register fpReg = x5;
Register opcode = w6;
Register bcStub = x7;
Label dispatch;
__ Ldr(fpReg, MemoryOperand(glue, JSThread::GlueData::GetLastFpOffset(false)));
__ Cmp(fpReg, Immediate(0));
__ B(Condition::EQ, &dispatch);
__ Mov(sp, fpReg);
__ Bind(&dispatch);
{
__ Ldrb(opcode, MemoryOperand(pc, 0));
__ Add(bcStub, glue, Operand(opcode, UXTW, FRAME_SLOT_SIZE_LOG2));
__ Ldr(bcStub, MemoryOperand(bcStub, JSThread::GlueData::GetBCStubEntriesOffset(false)));
__ Br(bcStub);
}
}
void AsmInterpreterCall::ResumeUncaughtFrameAndReturn(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeUncaughtFrameAndReturn));
Register glue = x19;
Register fpReg = x5;
Register acc = x20;
Register cppRet = x0;
__ Ldr(fpReg, MemoryOperand(glue, JSThread::GlueData::GetLastFpOffset(false)));
__ Mov(sp, fpReg);
__ Mov(cppRet, acc);
__ RestoreFpAndLr();
__ Ret();
}
void AsmInterpreterCall::ResumeRspAndRollback(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeRspAndRollback));
Register glueRegister = __ GlueRegister();
Register rsp = sp;
Register currentSp = fp;
Register pc = x20;
Register jumpSizeRegister = x25;
Register ret = x23;
Register opcode = w6;
Register bcStub = x7;
Register fpReg = x8;
int64_t fpOffset = static_cast<int64_t>(AsmInterpretedFrame::GetFpOffset(false))
- static_cast<int64_t>(AsmInterpretedFrame::GetSize(false));
int64_t spOffset = static_cast<int64_t>(AsmInterpretedFrame::GetBaseOffset(false))
- static_cast<int64_t>(AsmInterpretedFrame::GetSize(false));
int64_t funcOffset = static_cast<int64_t>(AsmInterpretedFrame::GetFunctionOffset(false))
- static_cast<int64_t>(AsmInterpretedFrame::GetSize(false));
ASSERT(fpOffset < 0);
ASSERT(spOffset < 0);
ASSERT(funcOffset < 0);
__ Ldur(fpReg, MemoryOperand(currentSp, fpOffset));
__ Ldur(ret, MemoryOperand(currentSp, funcOffset));
__ Ldur(currentSp, MemoryOperand(currentSp, spOffset));
__ Add(pc, pc, Operand(jumpSizeRegister, LSL, 0));
__ Ldrb(opcode, MemoryOperand(pc, 0));
__ Mov(rsp, fpReg);
__ Add(bcStub, glueRegister, Operand(opcode, UXTW, FRAME_SLOT_SIZE_LOG2));
__ Ldr(bcStub, MemoryOperand(bcStub, JSThread::GlueData::GetBCStubEntriesOffset(false)));
__ Br(bcStub);
}
void AsmInterpreterCall::CallGetter(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(CallGetter));
Label target;
PushAsmInterpBridgeFrame(assembler);
__ Bl(&target);
PopAsmInterpBridgeFrame(assembler);
__ Ret();
__ Bind(&target);
{
JSCallCommonEntry(assembler, JSCallMode::CALL_GETTER, FrameTransitionType::OTHER_TO_OTHER);
}
}
void AsmInterpreterCall::CallSetter(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(CallSetter));
Label target;
PushAsmInterpBridgeFrame(assembler);
__ Bl(&target);
PopAsmInterpBridgeFrame(assembler);
__ Ret();
__ Bind(&target);
{
JSCallCommonEntry(assembler, JSCallMode::CALL_SETTER, FrameTransitionType::OTHER_TO_OTHER);
}
}
void AsmInterpreterCall::CallContainersArgs2(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(CallContainersArgs2));
Label target;
PushAsmInterpBridgeFrame(assembler);
__ Bl(&target);
PopAsmInterpBridgeFrame(assembler);
__ Ret();
__ Bind(&target);
{
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG2_WITH_RETURN,
FrameTransitionType::OTHER_TO_OTHER);
}
}
void AsmInterpreterCall::CallContainersArgs3(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(CallContainersArgs3));
Label target;
PushAsmInterpBridgeFrame(assembler);
__ Bl(&target);
PopAsmInterpBridgeFrame(assembler);
__ Ret();
__ Bind(&target);
{
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG3_WITH_RETURN,
FrameTransitionType::OTHER_TO_OTHER);
}
}
void AsmInterpreterCall::CallReturnWithArgv(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(CallReturnWithArgv));
Label target;
PushAsmInterpBridgeFrame(assembler);
__ Bl(&target);
PopAsmInterpBridgeFrame(assembler);
__ Ret();
__ Bind(&target);
{
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARGV_WITH_RETURN,
FrameTransitionType::OTHER_TO_OTHER);
}
}
void AsmInterpreterCall::PreserveMostCall(ExtendedAssembler* assembler)
{
{
__ Stp(fp, lr, MemoryOperand(sp, -DOUBLE_SLOT_SIZE, PREINDEX));
__ Stp(x0, xzr, MemoryOperand(sp, -DOUBLE_SLOT_SIZE, PREINDEX));
__ Add(fp, sp, Immediate(DOUBLE_SLOT_SIZE));
}
int32_t PreserveRegPairIndex = 9;
__ Sub(sp, sp, Immediate(DOUBLE_SLOT_SIZE * PreserveRegPairIndex));
__ Stp(x1, x2, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (--PreserveRegPairIndex)));
__ Stp(x3, x4, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (--PreserveRegPairIndex)));
__ Stp(x5, x6, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (--PreserveRegPairIndex)));
__ Stp(x7, x8, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (--PreserveRegPairIndex)));
__ Stp(x9, x10, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (--PreserveRegPairIndex)));
__ Stp(x11, x12, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (--PreserveRegPairIndex)));
__ Stp(x13, x14, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (--PreserveRegPairIndex)));
__ Stp(x16, x17, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (--PreserveRegPairIndex)));
__ Str(x18, MemoryOperand(sp, FRAME_SLOT_SIZE));
__ Blr(x15);
__ Ldr(x18, MemoryOperand(sp, FRAME_SLOT_SIZE));
__ Ldp(x16, x17, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (PreserveRegPairIndex++)));
__ Ldp(x13, x14, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (PreserveRegPairIndex++)));
__ Ldp(x11, x12, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (PreserveRegPairIndex++)));
__ Ldp(x9, x10, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (PreserveRegPairIndex++)));
__ Ldp(x7, x8, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (PreserveRegPairIndex++)));
__ Ldp(x5, x6, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (PreserveRegPairIndex++)));
__ Ldp(x3, x4, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (PreserveRegPairIndex++)));
__ Ldp(x1, x2, MemoryOperand(sp, DOUBLE_SLOT_SIZE * (PreserveRegPairIndex++)));
__ Ldr(x0, MemoryOperand(sp, DOUBLE_SLOT_SIZE * PreserveRegPairIndex));
{
__ Add(sp, sp, Immediate(DOUBLE_SLOT_SIZE * PreserveRegPairIndex +
FRAME_SLOT_SIZE + FRAME_SLOT_SIZE));
__ Ldp(fp, lr, MemoryOperand(sp, DOUBLE_SLOT_SIZE, AddrMode::POSTINDEX));
__ Ret();
}
}
void AsmInterpreterCall::ASMFastWriteBarrier(ExtendedAssembler* assembler)
{
ASSERT(IN_YOUNG_SPACE < SHARED_SPACE_BEGIN && SHARED_SPACE_BEGIN <= SHARED_SWEEPABLE_SPACE_BEGIN &&
SHARED_SWEEPABLE_SPACE_END < IN_SHARED_READ_ONLY_SPACE && IN_SHARED_READ_ONLY_SPACE == HEAP_SPACE_END);
__ BindAssemblerStub(RTSTUB_ID(ASMFastWriteBarrier));
Label needCall;
Label checkMark;
Label needCallNotShare;
Label needShareBarrier;
Label valueNotShare;
Label valueMaybeSweepableShare;
{
__ And(x15, x3, LogicalImmediate::Create(~(JSTaggedValue::TAG_MARK | DEFAULT_REGION_MASK), X_REG_SIZE));
__ Ldrb(w15, MemoryOperand(x15, 0));
__ Cmp(w15, Immediate(SHARED_SWEEPABLE_SPACE_BEGIN));
__ B(GE, &valueMaybeSweepableShare);
}
#if USE_STICKY_CMS_GC
__ Bind(&valueNotShare);
{
__ And(x15, x3, LogicalImmediate::Create(~JSTaggedValue::TAG_WEAK, X_REG_SIZE));
__ Ldr(x15, MemoryOperand(x15, 0));
static_assert(TaggedStateWord::OLD_STATE == (0ULL << TaggedStateWord::ADDRESS_WIDTH));
static_assert(TaggedStateWord::YOUNG_STATE == (1ULL << TaggedStateWord::ADDRESS_WIDTH));
__ Tbz(x15, static_cast<uint32_t>(TaggedStateWord::ADDRESS_WIDTH), &checkMark);
__ Ldr(x15, MemoryOperand(x1, 0));
__ Tbz(x15, static_cast<uint32_t>(TaggedStateWord::ADDRESS_WIDTH), &needCallNotShare);
}
#else
__ Bind(&valueNotShare);
{
__ Cmp(w15, Immediate(RegionSpaceFlag::IN_YOUNG_SPACE));
__ B(NE, &checkMark);
__ And(x15, x1, LogicalImmediate::Create(~(JSTaggedValue::TAG_MARK | DEFAULT_REGION_MASK), X_REG_SIZE));
__ Ldrb(w15, MemoryOperand(x15, 0));
__ Cmp(w15, Immediate(RegionSpaceFlag::IN_YOUNG_SPACE));
__ B(NE, &needCallNotShare);
}
#endif
__ Bind(&checkMark);
{
__ Mov(x15, Immediate(JSThread::GlueData::GetGCStateBitFieldOffset(false)));
__ Ldrb(w15, MemoryOperand(x0, x15, UXTX));
__ Tst(w15, LogicalImmediate::Create(JSThread::CONCURRENT_MARKING_BITFIELD_MASK, W_REG_SIZE));
__ B(NE, &needCallNotShare);
__ Ret();
}
__ Bind(&valueMaybeSweepableShare);
{
__ Cmp(w15, Immediate(RegionSpaceFlag::IN_SHARED_READ_ONLY_SPACE));
__ B(NE, &needShareBarrier);
__ Ret();
}
__ Bind(&needCallNotShare);
{
int32_t NonSValueBarrier = static_cast<int32_t>(JSThread::GlueData::GetCOStubEntriesOffset(false)) +
kungfu::CommonStubCSigns::SetNonSValueWithBarrier * FRAME_SLOT_SIZE;
__ Mov(x15, Immediate(NonSValueBarrier));
}
__ Bind(&needCall);
{
__ Ldr(x15, MemoryOperand(x0, x15, UXTX));
PreserveMostCall(assembler);
}
__ Bind(&needShareBarrier);
{
ASMFastSharedWriteBarrier(assembler, needCall);
}
}
void AsmInterpreterCall::LoadBarrierCopyBack(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(LoadBarrierCopyBack));
Label copyBackTable;
int ldrOffset = -4;
__ Ldur(w1, MemoryOperand(x30, ldrOffset));
__ And(w1, w1, LogicalImmediate::Create(0x1F, W_REG_SIZE));
__ Adr(x2, ©BackTable);
int copyBackItemInsnShift = 3;
__ Add(x2, x2, Operand(w1, UXTW, copyBackItemInsnShift));
__ Br(x2);
__ Bind(©BackTable);
{
auto MovX0ToXregAndBlLr = [&assembler](Register xreg) {
__ Mov(xreg, x0);
__ Br(lr);
};
MovX0ToXregAndBlLr(x0);
MovX0ToXregAndBlLr(x1);
MovX0ToXregAndBlLr(x2);
MovX0ToXregAndBlLr(x3);
MovX0ToXregAndBlLr(x4);
MovX0ToXregAndBlLr(x5);
MovX0ToXregAndBlLr(x6);
MovX0ToXregAndBlLr(x7);
MovX0ToXregAndBlLr(x8);
MovX0ToXregAndBlLr(x9);
MovX0ToXregAndBlLr(x10);
MovX0ToXregAndBlLr(x11);
MovX0ToXregAndBlLr(x12);
MovX0ToXregAndBlLr(x13);
MovX0ToXregAndBlLr(x14);
MovX0ToXregAndBlLr(x15);
MovX0ToXregAndBlLr(x16);
MovX0ToXregAndBlLr(x17);
MovX0ToXregAndBlLr(x18);
MovX0ToXregAndBlLr(x19);
MovX0ToXregAndBlLr(x20);
MovX0ToXregAndBlLr(x21);
MovX0ToXregAndBlLr(x22);
MovX0ToXregAndBlLr(x23);
MovX0ToXregAndBlLr(x24);
MovX0ToXregAndBlLr(x25);
MovX0ToXregAndBlLr(x26);
MovX0ToXregAndBlLr(x27);
MovX0ToXregAndBlLr(x28);
MovX0ToXregAndBlLr(x29);
MovX0ToXregAndBlLr(x30);
MovX0ToXregAndBlLr(sp);
}
}
void AsmInterpreterCall::ASMFastSharedWriteBarrier(ExtendedAssembler* assembler, Label& needCall)
{
Label checkBarrierForSharedValue;
Label restoreScratchRegister;
Label callSharedBarrier;
{
__ And(x15, x1, LogicalImmediate::Create(~(JSTaggedValue::TAG_MARK | DEFAULT_REGION_MASK), X_REG_SIZE));
__ Ldrb(w15, MemoryOperand(x15, 0));
__ Cmp(w15, Immediate(RegionSpaceFlag::SHARED_SPACE_BEGIN));
__ B(GE, &checkBarrierForSharedValue);
}
{
__ And(x15, x1, LogicalImmediate::Create(~(JSTaggedValue::TAG_MARK | DEFAULT_REGION_MASK), X_REG_SIZE));
__ Ldr(x15, MemoryOperand(x15, Region::PackedData::GetLocalToShareSetOffset(false)));
__ Cbz(x15, &callSharedBarrier);
}
{
{
__ Stp(x16, x17, MemoryOperand(sp, -DOUBLE_SLOT_SIZE, PREINDEX));
}
__ And(x16, x1, LogicalImmediate::Create(DEFAULT_REGION_MASK, X_REG_SIZE));
__ Add(x16, x16, Operand(x2));
__ Ubfm(x17, x16, TAGGED_TYPE_SIZE_LOG, TAGGED_TYPE_SIZE_LOG + GCBitset::BIT_PER_WORD_LOG2 - 1);
__ Ubfm(x16, x16, TAGGED_TYPE_SIZE_LOG + GCBitset::BIT_PER_WORD_LOG2,
sizeof(uint32_t) * GCBitset::BIT_PER_BYTE + TAGGED_TYPE_SIZE_LOG - 1);
__ Add(x15, x15, Operand(x16, LSL, GCBitset::BYTE_PER_WORD_LOG2));
__ Add(x15, x15, Immediate(RememberedSet::GCBITSET_DATA_OFFSET));
__ Mov(w16, Immediate(1));
__ Lsl(w17, w16, w17);
__ Ldr(w16, MemoryOperand(x15, 0));
__ Tst(w16, w17);
__ B(NE, &restoreScratchRegister);
__ Orr(w16, w16, w17);
__ Str(w16, MemoryOperand(x15, 0));
}
__ Bind(&restoreScratchRegister);
{
__ Ldp(x16, x17, MemoryOperand(sp, DOUBLE_SLOT_SIZE, POSTINDEX));
}
__ Bind(&checkBarrierForSharedValue);
{
__ Mov(x15, Immediate(JSThread::GlueData::GetSharedGCStateBitFieldOffset(false)));
__ Ldrb(w15, MemoryOperand(x0, x15, UXTX));
static_assert(JSThread::SHARED_CONCURRENT_MARKING_BITFIELD_MASK == 1 && "Tbnz can't handle other bit mask");
__ Tbnz(w15, 0, &callSharedBarrier);
__ Ret();
}
__ Bind(&callSharedBarrier);
{
int32_t SValueBarrierOffset = static_cast<int32_t>(JSThread::GlueData::GetCOStubEntriesOffset(false)) +
kungfu::CommonStubCSigns::SetSValueWithBarrier * FRAME_SLOT_SIZE;
__ Mov(x15, Immediate(SValueBarrierOffset));
__ B(&needCall);
}
}
void AsmInterpreterCall::GeneratorReEnterAsmInterp(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(GeneratorReEnterAsmInterp));
Label target;
size_t begin = __ GetCurrentPosition();
PushAsmInterpEntryFrame(assembler);
__ Bl(&target);
PopAsmInterpEntryFrame(assembler);
size_t end = __ GetCurrentPosition();
if ((end - begin) != FrameCompletionPos::ARM64EntryFrameDuration) {
LOG_COMPILER(FATAL) << (end - begin) << " != " << FrameCompletionPos::ARM64EntryFrameDuration
<< "This frame has been modified, and the offset EntryFrameDuration should be updated too.";
}
__ Ret();
__ Bind(&target);
{
GeneratorReEnterAsmInterpDispatch(assembler);
}
}
void AsmInterpreterCall::GeneratorReEnterAsmInterpDispatch(ExtendedAssembler *assembler)
{
Label pushFrameState;
Label stackOverflow;
Register glue = __ GlueRegister();
Register contextRegister = x1;
Register pc = x8;
Register prevSpRegister = fp;
Register callTarget = x4;
Register method = x5;
Register temp = x6;
Register currentSlotRegister = x7;
Register fpRegister = x9;
Register thisRegister = x25;
Register nRegsRegister = w26;
Register regsArrayRegister = x27;
Register newSp = x28;
__ Ldr(callTarget, MemoryOperand(contextRegister, GeneratorContext::GENERATOR_METHOD_OFFSET));
__ Ldr(method, MemoryOperand(callTarget, JSFunctionBase::METHOD_OFFSET));
__ PushFpAndLr();
__ Mov(fpRegister, sp);
__ Mov(currentSlotRegister, sp);
__ Ldr(temp, MemoryOperand(glue, JSThread::GlueData::GetStackLimitOffset(false)));
__ Mov(sp, temp);
__ Ldr(nRegsRegister, MemoryOperand(contextRegister, GeneratorContext::GENERATOR_NREGS_OFFSET));
__ Ldr(thisRegister, MemoryOperand(contextRegister, GeneratorContext::GENERATOR_THIS_OFFSET));
__ Ldr(regsArrayRegister, MemoryOperand(contextRegister, GeneratorContext::GENERATOR_REGS_ARRAY_OFFSET));
__ Add(regsArrayRegister, regsArrayRegister, Immediate(TaggedArray::DATA_OFFSET));
PushArgsWithArgv(assembler, glue, nRegsRegister, regsArrayRegister, temp,
currentSlotRegister, &pushFrameState, &stackOverflow);
__ Bind(&pushFrameState);
__ Mov(newSp, currentSlotRegister);
PushGeneratorFrameState(assembler, prevSpRegister, fpRegister, currentSlotRegister, callTarget, thisRegister,
method, contextRegister, pc, temp);
__ Align16(currentSlotRegister);
__ Mov(sp, currentSlotRegister);
CallBCStub(assembler, newSp, glue, callTarget, method, pc, temp);
__ Bind(&stackOverflow);
{
ThrowStackOverflowExceptionAndReturn(assembler, glue, fpRegister, temp);
}
}
void AsmInterpreterCall::PushCallThis(ExtendedAssembler *assembler,
JSCallMode mode, Label *stackOverflow, FrameTransitionType type)
{
Register callFieldRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
Register callTargetRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_TARGET);
Register thisRegister = __ AvailableRegister2();
Register currentSlotRegister = __ AvailableRegister3();
Label pushVregs;
Label pushNewTarget;
Label pushCallTarget;
bool haveThis = kungfu::AssemblerModule::JSModeHaveThisArg(mode);
bool haveNewTarget = kungfu::AssemblerModule::JSModeHaveNewTargetArg(mode);
if (!haveThis) {
__ Mov(thisRegister, Immediate(JSTaggedValue::VALUE_UNDEFINED));
} else {
Register thisArgRegister = GetThisRegsiter(assembler, mode, thisRegister);
if (thisRegister.GetId() != thisArgRegister.GetId()) {
__ Mov(thisRegister, thisArgRegister);
}
}
__ Tst(callFieldRegister, LogicalImmediate::Create(CALL_TYPE_MASK, X_REG_SIZE));
__ B(Condition::EQ, &pushVregs);
__ Tbz(callFieldRegister, Method::HaveThisBit::START_BIT, &pushNewTarget);
if (!haveThis) {
[[maybe_unused]] TempRegister1Scope scope1(assembler);
Register tempRegister = __ TempRegister1();
__ Mov(tempRegister, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ Str(tempRegister, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
} else {
__ Str(thisRegister, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
}
__ Bind(&pushNewTarget);
{
__ Tbz(callFieldRegister, Method::HaveNewTargetBit::START_BIT, &pushCallTarget);
if (!haveNewTarget) {
[[maybe_unused]] TempRegister1Scope scope1(assembler);
Register newTarget = __ TempRegister1();
__ Mov(newTarget, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ Str(newTarget, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
} else {
[[maybe_unused]] TempRegister1Scope scope1(assembler);
Register defaultRegister = __ TempRegister1();
Register newTargetRegister = GetNewTargetRegsiter(assembler, mode, defaultRegister);
__ Str(newTargetRegister, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
}
}
__ Bind(&pushCallTarget);
{
__ Tbz(callFieldRegister, Method::HaveFuncBit::START_BIT, &pushVregs);
__ Str(callTargetRegister, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
}
__ Bind(&pushVregs);
{
PushVregs(assembler, stackOverflow, type);
}
}
void AsmInterpreterCall::PushVregs(ExtendedAssembler *assembler,
Label *stackOverflow, FrameTransitionType type)
{
Register glue = __ GlueRegister();
Register prevSpRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::SP);
Register callTargetRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_TARGET);
Register methodRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::METHOD);
Register callFieldRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
Register fpRegister = __ AvailableRegister1();
Register thisRegister = __ AvailableRegister2();
Register currentSlotRegister = __ AvailableRegister3();
Label pushFrameStateAndCall;
[[maybe_unused]] TempRegister1Scope scope1(assembler);
Register tempRegister = __ TempRegister1();
Register newSpRegister = __ AvailableRegister4();
Register numVregsRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
GetNumVregsFromCallField(assembler, callFieldRegister, numVregsRegister);
PushUndefinedWithArgc(assembler, glue, numVregsRegister, tempRegister, currentSlotRegister, &pushFrameStateAndCall,
stackOverflow);
__ Bind(&pushFrameStateAndCall);
{
__ Mov(newSpRegister, currentSlotRegister);
[[maybe_unused]] TempRegister2Scope scope2(assembler);
Register pcRegister = __ TempRegister2();
PushFrameState(assembler, prevSpRegister, fpRegister, currentSlotRegister, callTargetRegister, thisRegister,
methodRegister, pcRegister, tempRegister);
__ Align16(currentSlotRegister);
__ Mov(sp, currentSlotRegister);
if (type == FrameTransitionType::OTHER_TO_BASELINE_CHECK ||
type == FrameTransitionType::BASELINE_TO_BASELINE_CHECK) {
Label baselineCodeUndefined;
__ Ldr(tempRegister, MemoryOperand(callTargetRegister, JSFunction::BASELINECODE_OFFSET));
__ Cmp(tempRegister, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ B(Condition::EQ, &baselineCodeUndefined);
__ Cmp(tempRegister, Immediate(JSTaggedValue::VALUE_HOLE));
__ B(Condition::EQ, &baselineCodeUndefined);
if (MachineCode::FUNCADDR_OFFSET % 8 == 0) {
__ Ldr(tempRegister, MemoryOperand(tempRegister, MachineCode::FUNCADDR_OFFSET));
} else {
ASSERT(MachineCode::FUNCADDR_OFFSET < 256);
__ Ldur(tempRegister, MemoryOperand(tempRegister, MachineCode::FUNCADDR_OFFSET));
}
if (glue != x19) {
__ Mov(x19, glue);
}
if (methodRegister != x21) {
__ Mov(x21, methodRegister);
}
__ Mov(currentSlotRegister, Immediate(BASELINEJIT_PC_FLAG));
__ Stur(currentSlotRegister, MemoryOperand(newSpRegister, -3 * FRAME_SLOT_SIZE));
__ Mov(x29, newSpRegister);
__ Br(tempRegister);
__ Bind(&baselineCodeUndefined);
}
DispatchCall(assembler, pcRegister, newSpRegister);
}
}
void AsmInterpreterCall::DispatchCall(ExtendedAssembler *assembler, Register pcRegister,
Register newSpRegister, Register accRegister, bool hasException)
{
Register glueRegister = __ GlueRegister();
Register callTargetRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_TARGET);
Register methodRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::METHOD);
if (glueRegister != x19) {
__ Mov(x19, glueRegister);
}
__ Ldrh(w24, MemoryOperand(methodRegister, Method::LITERAL_INFO_OFFSET));
if (!accRegister.IsValid()) {
__ Mov(x23, Immediate(JSTaggedValue::VALUE_HOLE));
} else {
ASSERT(accRegister == x23);
}
__ Ldr(x22, MemoryOperand(callTargetRegister, JSFunction::RAW_PROFILE_TYPE_INFO_OFFSET));
__ Ldr(x22, MemoryOperand(x22, ProfileTypeInfoCell::VALUE_OFFSET));
__ Ldr(x21, MemoryOperand(methodRegister, Method::CONSTANT_POOL_OFFSET));
__ Mov(x20, pcRegister);
__ Mov(fp, newSpRegister);
Register bcIndexRegister = __ AvailableRegister1();
Register tempRegister = __ AvailableRegister2();
if (hasException) {
__ Mov(bcIndexRegister.W(), Immediate(kungfu::BytecodeStubCSigns::ID_ExceptionHandler));
} else {
__ Ldrb(bcIndexRegister.W(), MemoryOperand(pcRegister, 0));
}
__ Add(tempRegister, glueRegister, Operand(bcIndexRegister.W(), UXTW, FRAME_SLOT_SIZE_LOG2));
__ Ldr(tempRegister, MemoryOperand(tempRegister, JSThread::GlueData::GetBCStubEntriesOffset(false)));
__ UpdateGlueAndReadBarrier(glueRegister);
__ Br(tempRegister);
}
void AsmInterpreterCall::PushFrameState(ExtendedAssembler *assembler, Register prevSp, Register fpReg,
Register currentSlot, Register callTarget, Register thisObj, Register method, Register pc, Register op)
{
__ Mov(op, Immediate(static_cast<int32_t>(FrameType::ASM_INTERPRETER_FRAME)));
__ Stp(prevSp, op, MemoryOperand(currentSlot, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Ldr(pc, MemoryOperand(method, Method::NATIVE_POINTER_OR_BYTECODE_ARRAY_OFFSET));
__ Stp(fpReg, pc, MemoryOperand(currentSlot, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Ldr(op, MemoryOperand(callTarget, JSFunction::LEXICAL_ENV_OFFSET));
__ Stp(op, xzr, MemoryOperand(currentSlot,
-2 * FRAME_SLOT_SIZE,
AddrMode::PREINDEX));
__ Mov(op, Immediate(JSTaggedValue::VALUE_HOLE));
__ Stp(thisObj, op, MemoryOperand(currentSlot, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Str(callTarget, MemoryOperand(currentSlot, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
}
void AsmInterpreterCall::GetNumVregsFromCallField(ExtendedAssembler *assembler, Register callField, Register numVregs)
{
__ Mov(numVregs, callField);
__ Lsr(numVregs, numVregs, Method::NumVregsBits::START_BIT);
__ And(numVregs.W(), numVregs.W(), LogicalImmediate::Create(
Method::NumVregsBits::Mask() >> Method::NumVregsBits::START_BIT, W_REG_SIZE));
}
void AsmInterpreterCall::GetDeclaredNumArgsFromCallField(ExtendedAssembler *assembler, Register callField,
Register declaredNumArgs)
{
__ Mov(declaredNumArgs, callField);
__ Lsr(declaredNumArgs, declaredNumArgs, Method::NumArgsBits::START_BIT);
__ And(declaredNumArgs.W(), declaredNumArgs.W(), LogicalImmediate::Create(
Method::NumArgsBits::Mask() >> Method::NumArgsBits::START_BIT, W_REG_SIZE));
}
void AsmInterpreterCall::PushAsmInterpEntryFrame(ExtendedAssembler *assembler)
{
Register glue = __ GlueRegister();
size_t begin = __ GetCurrentPosition();
if (!assembler->FromInterpreterHandler()) {
__ CalleeSave();
}
[[maybe_unused]] TempRegister1Scope scope1(assembler);
Register prevFrameRegister = __ TempRegister1();
[[maybe_unused]] TempRegister2Scope scope2(assembler);
Register frameTypeRegister = __ TempRegister2();
__ PushFpAndLr();
__ Ldr(prevFrameRegister, MemoryOperand(glue, JSThread::GlueData::GetLeaveFrameOffset(false)));
__ Mov(frameTypeRegister, Immediate(static_cast<int64_t>(FrameType::ASM_INTERPRETER_ENTRY_FRAME)));
__ Stp(prevFrameRegister, frameTypeRegister, MemoryOperand(sp, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Stp(glue, xzr, MemoryOperand(sp, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
if (!assembler->FromInterpreterHandler()) {
size_t end = __ GetCurrentPosition();
if ((end - begin) != FrameCompletionPos::ARM64CppToAsmInterp) {
LOG_COMPILER(FATAL) << (end - begin) << " != " << FrameCompletionPos::ARM64CppToAsmInterp
<< "This frame has been modified, and the offset CppToAsmInterp should be updated too.";
}
}
__ Add(fp, sp, Immediate(4 * FRAME_SLOT_SIZE));
}
void AsmInterpreterCall::PopAsmInterpEntryFrame(ExtendedAssembler *assembler)
{
[[maybe_unused]] TempRegister1Scope scope1(assembler);
Register prevFrameRegister = __ TempRegister1();
[[maybe_unused]] TempRegister2Scope scope2(assembler);
Register glue = __ TempRegister2();
__ Ldp(glue, xzr, MemoryOperand(sp, 2 * FRAME_SLOT_SIZE, AddrMode::POSTINDEX));
__ Ldr(prevFrameRegister, MemoryOperand(sp, 0));
__ Str(prevFrameRegister, MemoryOperand(glue, JSThread::GlueData::GetLeaveFrameOffset(false)));
__ Ldp(prevFrameRegister, xzr, MemoryOperand(sp, 2 * FRAME_SLOT_SIZE, AddrMode::POSTINDEX));
size_t begin = __ GetCurrentPosition();
__ RestoreFpAndLr();
if (!assembler->FromInterpreterHandler()) {
__ CalleeRestore();
size_t end = __ GetCurrentPosition();
if ((end - begin) != FrameCompletionPos::ARM64AsmInterpToCpp) {
LOG_COMPILER(FATAL) << (end - begin) << " != " << FrameCompletionPos::ARM64AsmInterpToCpp
<< "This frame has been modified, and the offset AsmInterpToCpp should be updated too.";
}
}
}
void AsmInterpreterCall::PushGeneratorFrameState(ExtendedAssembler *assembler, Register &prevSpRegister,
Register &fpRegister, Register ¤tSlotRegister, Register &callTargetRegister, Register &thisRegister,
Register &methodRegister, Register &contextRegister, Register &pcRegister, Register &operatorRegister)
{
__ Mov(operatorRegister, Immediate(static_cast<int64_t>(FrameType::ASM_INTERPRETER_FRAME)));
__ Stp(prevSpRegister, operatorRegister,
MemoryOperand(currentSlotRegister, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Ldr(pcRegister, MemoryOperand(methodRegister, Method::NATIVE_POINTER_OR_BYTECODE_ARRAY_OFFSET));
__ Ldr(operatorRegister, MemoryOperand(contextRegister, GeneratorContext::GENERATOR_NREGS_OFFSET));
__ Lsr(operatorRegister, operatorRegister, 32);
__ Add(pcRegister, operatorRegister, pcRegister);
__ Stp(fpRegister, pcRegister, MemoryOperand(currentSlotRegister, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Str(xzr, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Ldr(operatorRegister, MemoryOperand(contextRegister, GeneratorContext::GENERATOR_LEXICALENV_OFFSET));
__ Str(operatorRegister, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Ldr(operatorRegister, MemoryOperand(contextRegister, GeneratorContext::GENERATOR_ACC_OFFSET));
__ Str(operatorRegister, MemoryOperand(currentSlotRegister, -FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Stp(callTargetRegister, thisRegister,
MemoryOperand(currentSlotRegister, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
}
void AsmInterpreterCall::CallBCStub(ExtendedAssembler *assembler, Register &newSp, Register &glue,
Register &callTarget, Register &method, Register &pc, Register &temp)
{
__ Mov(x19, glue);
__ Mov(fp, newSp);
__ Mov(x20, pc);
__ Ldr(x21, MemoryOperand(method, Method::CONSTANT_POOL_OFFSET));
__ Ldr(x22, MemoryOperand(callTarget, JSFunction::RAW_PROFILE_TYPE_INFO_OFFSET));
__ Ldr(x22, MemoryOperand(x22, ProfileTypeInfoCell::VALUE_OFFSET));
__ Mov(x23, Immediate(JSTaggedValue::Hole().GetRawData()));
__ Ldr(x24, MemoryOperand(method, Method::LITERAL_INFO_OFFSET));
__ Ldrb(temp.W(), MemoryOperand(pc, 0));
__ Add(temp, glue, Operand(temp.W(), UXTW, FRAME_SLOT_SIZE_LOG2));
__ Ldr(temp, MemoryOperand(temp, JSThread::GlueData::GetBCStubEntriesOffset(false)));
__ UpdateGlueAndReadBarrier(glue);
__ Br(temp);
}
void AsmInterpreterCall::CallNativeEntry(ExtendedAssembler *assembler, bool isJSFunction)
{
Label callFastBuiltin;
Label callNativeBuiltin;
Register glue = x0;
Register argv = x5;
Register function = x1;
Register nativeCode = x7;
Register temp = x9;
if (isJSFunction) {
Register callFieldRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
__ Ldr(nativeCode, MemoryOperand(function, JSFunctionBase::CODE_ENTRY_OFFSET));
__ Tbnz(callFieldRegister, Method::IsFastBuiltinBit::START_BIT, &callFastBuiltin);
} else {
Register method = x2;
__ Ldr(nativeCode, MemoryOperand(method, Method::NATIVE_POINTER_OR_BYTECODE_ARRAY_OFFSET));
}
__ Bind(&callNativeBuiltin);
if (isJSFunction) {
Register lexicalEnv = temp;
Label next;
__ Ldr(lexicalEnv, MemoryOperand(function, JSFunction::LEXICAL_ENV_OFFSET));
__ Cmp(lexicalEnv, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ B(Condition::EQ, &next);
__ Str(lexicalEnv, MemoryOperand(glue, JSThread::GlueData::GetCurrentEnvOffset(false)));
__ Bind(&next);
}
__ Stp(function, xzr, MemoryOperand(sp, -2 * FRAME_SLOT_SIZE, AddrMode::PREINDEX));
__ Sub(sp, sp, Immediate(2 * FRAME_SLOT_SIZE));
PushBuiltinFrame(assembler, glue, FrameType::BUILTIN_ENTRY_FRAME, temp, argv);
__ Mov(temp, argv);
#ifdef ENABLE_CMC_IR_FIX_REGISTER
Register calleeSaveGlue = x28;
__ Mov(calleeSaveGlue, glue);
#endif
__ Sub(x0, temp, Immediate(2 * FRAME_SLOT_SIZE));
CallNativeInternal(assembler, nativeCode);
__ Add(sp, sp, Immediate(4 * FRAME_SLOT_SIZE));
__ Ret();
__ Bind(&callFastBuiltin);
CallFastBuiltin(assembler, &callNativeBuiltin);
}
void AsmInterpreterCall::CallFastBuiltin(ExtendedAssembler *assembler, Label *callNativeBuiltin)
{
Label dispatchTable[3];
Label callEntryAndRet;
Register glue = x0;
Register function = x1;
Register method = x2;
Register argc = x4;
Register argv = x5;
Register nativeCode = x7;
Register builtinId = __ AvailableRegister1();
Register temp = __ AvailableRegister2();
__ Ldr(builtinId, MemoryOperand(method, Method::EXTRA_LITERAL_INFO_OFFSET));
__ Lsr(builtinId.W(), builtinId.W(), Method::BuiltinIdBits::START_BIT);
__ And(builtinId.W(), builtinId.W(),
LogicalImmediate::Create((1LU << Method::BuiltinIdBits::SIZE) - 1, W_REG_SIZE));
__ Cmp(builtinId.W(), Immediate(BUILTINS_STUB_ID(BUILTINS_CONSTRUCTOR_STUB_FIRST)));
__ B(Condition::GE, callNativeBuiltin);
__ Cmp(argc, Immediate(3));
__ B(Condition::HI, callNativeBuiltin);
__ Add(builtinId, glue, Operand(builtinId.W(), UXTW, FRAME_SLOT_SIZE_LOG2));
__ Ldr(builtinId, MemoryOperand(builtinId, JSThread::GlueData::GetBuiltinsStubEntriesOffset(false)));
PushAsmBridgeFrame(assembler);
__ Mov(temp, function);
__ Mov(x1, nativeCode);
__ Mov(x2, temp);
__ Mov(temp, argv);
__ Mov(x5, argc);
__ Ldr(x3, MemoryOperand(temp, FRAME_SLOT_SIZE));
__ Ldr(x4, MemoryOperand(temp, DOUBLE_SLOT_SIZE));
__ Cmp(x5, Immediate(0));
__ B(Condition::EQ, &dispatchTable[0]);
__ Cmp(x5, Immediate(1));
__ B(Condition::EQ, &dispatchTable[1]);
__ Cmp(x5, Immediate(2));
__ B(Condition::EQ, &dispatchTable[2]);
__ Ldr(x7, MemoryOperand(temp, QUINTUPLE_SLOT_SIZE));
__ Stp(x7, x7, MemoryOperand(sp, -DOUBLE_SLOT_SIZE, PREINDEX));
__ Ldp(x6, x7, MemoryOperand(temp, TRIPLE_SLOT_SIZE));
__ B(&callEntryAndRet);
__ Bind(&dispatchTable[0]);
{
__ Mov(x6, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ Mov(x7, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ Stp(x7, x7, MemoryOperand(sp, -DOUBLE_SLOT_SIZE, PREINDEX));
__ B(&callEntryAndRet);
}
__ Bind(&dispatchTable[1]);
{
__ Ldr(x6, MemoryOperand(temp, TRIPLE_SLOT_SIZE));
__ Mov(x7, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ Stp(x7, x7, MemoryOperand(sp, -DOUBLE_SLOT_SIZE, PREINDEX));
__ B(&callEntryAndRet);
}
__ Bind(&dispatchTable[2]);
{
__ Mov(x7, Immediate(JSTaggedValue::VALUE_UNDEFINED));
__ Stp(x7, x7, MemoryOperand(sp, -DOUBLE_SLOT_SIZE, PREINDEX));
__ Ldp(x6, x7, MemoryOperand(temp, TRIPLE_SLOT_SIZE));
}
__ Bind(&callEntryAndRet);
{
__ Blr(builtinId);
__ Add(sp, sp, Immediate(DOUBLE_SLOT_SIZE));
PopAsmBridgeFrame(assembler);
__ Ret();
}
}
void AsmInterpreterCall::ThrowStackOverflowExceptionAndReturn(ExtendedAssembler *assembler, Register glue,
Register fpReg, Register op)
{
if (fpReg != sp) {
__ Mov(sp, fpReg);
}
__ Mov(op, Immediate(kungfu::RuntimeStubCSigns::ID_ThrowStackOverflowException));
__ Add(op, glue, Operand(op, LSL, 3));
__ Ldr(op, MemoryOperand(op, JSThread::GlueData::GetRTStubEntriesOffset(false)));
if (glue != x0) {
__ Mov(x0, glue);
}
#ifdef ENABLE_CMC_IR_FIX_REGISTER
__ Mov(x28, glue);
#endif
__ Blr(op);
__ UpdateGlueAndReadBarrier();
__ RestoreFpAndLr();
__ Ret();
}
void AsmInterpreterCall::ThrowStackOverflowExceptionAndReturnToAsmInterpBridgeFrame(ExtendedAssembler *assembler,
Register glue, Register fpReg, Register op)
{
if (fpReg != sp) {
__ Mov(sp, fpReg);
}
if (glue != x0) {
__ Mov(x0, glue);
}
__ PushFpAndLr();
__ Mov(op, Immediate(static_cast<int64_t>(FrameType::ASM_BRIDGE_FRAME)));
__ Stp(x10, op, MemoryOperand(sp, -DOUBLE_SLOT_SIZE, PREINDEX));
__ Add(fp, sp, Immediate(DOUBLE_SLOT_SIZE));
__ Mov(op, Immediate(kungfu::RuntimeStubCSigns::ID_ThrowStackOverflowException));
__ Stp(op, xzr, MemoryOperand(sp, -DOUBLE_SLOT_SIZE, PREINDEX));
__ Mov(x10, Immediate(kungfu::RuntimeStubCSigns::ID_CallRuntime));
__ Add(x10, glue, Operand(x10, LSL, 3));
__ Ldr(x10, MemoryOperand(x10, JSThread::GlueData::GetRTStubEntriesOffset(false)));
__ Blr(x10);
__ Add(sp, sp, Immediate(2 * FRAME_SLOT_SIZE));
__ Ldr(x10, MemoryOperand(sp, FRAME_SLOT_SIZE, POSTINDEX));
__ Mov(sp, fp);
__ RestoreFpAndLr();
int32_t skipNum = static_cast<int32_t>(AsmInterpretedBridgeFrame::GetSize(false)) / FRAME_SLOT_SIZE + 18 - 1;
__ Add(sp, fp, Immediate(-skipNum * FRAME_SLOT_SIZE));
__ Ret();
}
#undef __
}