* Copyright (c) 2022-2024 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ecmascript/compiler/trampoline/x64/common_call.h"
#include "ecmascript/js_generator_object.h"
#include "ecmascript/message_string.h"
namespace panda::ecmascript::x64 {
#define __ assembler->
void AsmInterpreterCall::AsmInterpreterEntry(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(AsmInterpreterEntry));
Label target;
size_t begin = __ GetCurrentPosition();
PushAsmInterpEntryFrame(assembler);
__ Callq(&target);
PopAsmInterpEntryFrame(assembler);
size_t end = __ GetCurrentPosition();
if ((end - begin) != FrameCompletionPos::X64EntryFrameDuration) {
LOG_COMPILER(FATAL) << (end - begin) << " != " << FrameCompletionPos::X64EntryFrameDuration
<< "This frame has been modified, and the offset EntryFrameDuration should be updated too.";
}
__ Ret();
__ Bind(&target);
AsmInterpEntryDispatch(assembler);
}
void AsmInterpreterCall::GeneratorReEnterAsmInterp(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(GeneratorReEnterAsmInterp));
Label target;
size_t begin = __ GetCurrentPosition();
PushAsmInterpEntryFrame(assembler);
__ Callq(&target);
PopAsmInterpEntryFrame(assembler);
size_t end = __ GetCurrentPosition();
if ((end - begin) != FrameCompletionPos::X64EntryFrameDuration) {
LOG_COMPILER(FATAL) << (end - begin) << " != " << FrameCompletionPos::X64EntryFrameDuration
<< "This frame has been modified, and the offset EntryFrameDuration should be updated too.";
}
__ Ret();
__ Bind(&target);
GeneratorReEnterAsmInterpDispatch(assembler);
}
void AsmInterpreterCall::GeneratorReEnterAsmInterpDispatch(ExtendedAssembler *assembler)
{
Register glueRegister = __ GlueRegister();
Register contextRegister = rsi;
Register prevSpRegister = rbp;
Register callTargetRegister = r9;
Register methodRegister = rcx;
Register tempRegister = r11;
Register opRegister = r8;
__ Movq(Operand(rsi, GeneratorContext::GENERATOR_METHOD_OFFSET), callTargetRegister);
__ Movq(Operand(callTargetRegister, JSFunctionBase::METHOD_OFFSET), methodRegister);
Label stackOverflow;
Register fpRegister = r10;
__ Movq(rsp, fpRegister);
Register nRegsRegister = rdx;
Register regsArrayRegister = r12;
Register thisRegister = r15;
__ Movl(Operand(rsi, GeneratorContext::GENERATOR_NREGS_OFFSET), nRegsRegister);
__ Movq(Operand(rsi, GeneratorContext::GENERATOR_THIS_OFFSET), thisRegister);
__ Movq(Operand(rsi, GeneratorContext::GENERATOR_REGS_ARRAY_OFFSET), regsArrayRegister);
__ Addq(TaggedArray::DATA_OFFSET, regsArrayRegister);
PushArgsWithArgvAndCheckStack(assembler, glueRegister, nRegsRegister, regsArrayRegister, tempRegister, opRegister,
&stackOverflow);
Register newSpRegister = r8;
__ Movq(rsp, newSpRegister);
Register pcRegister = r12;
PushGeneratorFrameState(assembler, prevSpRegister, fpRegister, callTargetRegister, thisRegister, methodRegister,
contextRegister, pcRegister, tempRegister);
DispatchCall(assembler, pcRegister, newSpRegister, callTargetRegister, methodRegister);
__ Bind(&stackOverflow);
{
ThrowStackOverflowExceptionAndReturn(assembler, glueRegister, fpRegister, tempRegister);
}
}
void AsmInterpreterCall::AsmInterpEntryDispatch(ExtendedAssembler *assembler)
{
Label notJSFunction;
Label callNativeEntry;
Label callJSFunctionEntry;
Label notCallable;
Register glueRegister = rdi;
Register callTargetRegister = rsi;
Register argvRegister = r9;
Register bitFieldRegister = r12;
Register tempRegister = r11;
__ Movq(Operand(callTargetRegister, TaggedObject::HCLASS_OFFSET), tempRegister);
Register maskRegister = r12;
__ Movabs(TaggedObject::GC_STATE_MASK, maskRegister);
__ And(maskRegister, tempRegister);
__ Movq(Operand(tempRegister, JSHClass::BIT_FIELD_OFFSET), bitFieldRegister);
__ Cmpb(static_cast<int32_t>(JSType::JS_FUNCTION_FIRST), bitFieldRegister);
__ Jb(¬JSFunction);
__ Cmpb(static_cast<int32_t>(JSType::JS_FUNCTION_LAST), bitFieldRegister);
__ Jbe(&callJSFunctionEntry);
__ Bind(¬JSFunction);
{
__ Testq(static_cast<int64_t>(1ULL << JSHClass::CallableBit::START_BIT), bitFieldRegister);
__ Jz(¬Callable);
CallNativeEntry(assembler, false);
}
__ Bind(&callNativeEntry);
CallNativeEntry(assembler, true);
__ Bind(&callJSFunctionEntry);
{
Register callFieldRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
__ Btq(Method::IsNativeBit::START_BIT, callFieldRegister);
__ Jb(&callNativeEntry);
__ Leaq(Operand(argvRegister, NUM_MANDATORY_JSFUNC_ARGS * JSTaggedValue::TaggedTypeSize()),
argvRegister);
JSCallCommonEntry(assembler, JSCallMode::CALL_ENTRY, FrameTransitionType::OTHER_TO_BASELINE_CHECK);
}
__ Bind(¬Callable);
{
__ Movq(glueRegister, rax);
__ Pushq(0);
Register runtimeIdRegister = r12;
__ Movq(kungfu::RuntimeStubCSigns::ID_ThrowNotCallableException, runtimeIdRegister);
__ Pushq(runtimeIdRegister);
Register trampolineIdRegister = r12;
Register trampolineRegister = r10;
__ Movq(kungfu::RuntimeStubCSigns::ID_CallRuntime, trampolineIdRegister);
__ Movq(Operand(rax, trampolineIdRegister, Times8, JSThread::GlueData::GetRTStubEntriesOffset(false)),
trampolineRegister);
#ifdef ENABLE_CMC_IR_FIX_REGISTER
__ Movq(glueRegister, r15);
#endif
__ Callq(trampolineRegister);
__ UpdateReadBarrier();
__ Addq(16, rsp);
__ Ret();
}
}
void AsmInterpreterCall::PushFrameState(ExtendedAssembler *assembler, Register prevSpRegister, Register fpRegister,
Register callTargetRegister, Register thisRegister, Register methodRegister, Register pcRegister,
Register operatorRegister)
{
__ Pushq(static_cast<int32_t>(FrameType::ASM_INTERPRETER_FRAME));
__ Pushq(prevSpRegister);
__ Movq(Operand(methodRegister, Method::NATIVE_POINTER_OR_BYTECODE_ARRAY_OFFSET), pcRegister);
__ Pushq(pcRegister);
__ Pushq(fpRegister);
__ Pushq(0);
__ Movq(Operand(callTargetRegister, JSFunction::LEXICAL_ENV_OFFSET), operatorRegister);
__ Pushq(operatorRegister);
__ Pushq(JSTaggedValue::Hole().GetRawData());
__ Pushq(thisRegister);
__ Pushq(callTargetRegister);
}
void AsmInterpreterCall::PushGeneratorFrameState(ExtendedAssembler *assembler, Register prevSpRegister,
Register fpRegister, Register callTargetRegister, Register thisRegister, Register methodRegister,
Register contextRegister, Register pcRegister, Register operatorRegister)
{
__ Pushq(static_cast<int32_t>(FrameType::ASM_INTERPRETER_FRAME));
__ Pushq(prevSpRegister);
__ Movq(Operand(methodRegister, Method::NATIVE_POINTER_OR_BYTECODE_ARRAY_OFFSET), pcRegister);
__ Movl(Operand(contextRegister, GeneratorContext::GENERATOR_BC_OFFSET_OFFSET), operatorRegister);
__ Addq(operatorRegister, pcRegister);
__ Pushq(pcRegister);
__ Pushq(fpRegister);
__ Pushq(0);
__ Movq(Operand(contextRegister, GeneratorContext::GENERATOR_LEXICALENV_OFFSET), operatorRegister);
__ Pushq(operatorRegister);
__ Movq(Operand(contextRegister, GeneratorContext::GENERATOR_ACC_OFFSET), operatorRegister);
__ Pushq(operatorRegister);
__ Pushq(thisRegister);
__ Pushq(callTargetRegister);
}
void AsmInterpreterCall::PushAsmInterpEntryFrame(ExtendedAssembler *assembler)
{
size_t begin = __ GetCurrentPosition();
if (!assembler->FromInterpreterHandler()) {
__ PushCppCalleeSaveRegisters();
}
Register fpRegister = r10;
__ Pushq(rdi);
__ PushAlignBytes();
__ Movq(Operand(rdi, JSThread::GlueData::GetLeaveFrameOffset(false)), fpRegister);
__ Pushq(rbp);
__ Pushq(static_cast<int64_t>(FrameType::ASM_INTERPRETER_ENTRY_FRAME));
__ Pushq(fpRegister);
__ Pushq(0);
if (!assembler->FromInterpreterHandler()) {
size_t end = __ GetCurrentPosition();
if ((end - begin) != FrameCompletionPos::X64CppToAsmInterp) {
LOG_COMPILER(FATAL) << (end - begin) << " != " << FrameCompletionPos::X64CppToAsmInterp
<< "This frame has been modified, and the offset CppToAsmInterp should be updated too.";
}
}
__ Leaq(Operand(rsp, 3 * FRAME_SLOT_SIZE), rbp);
}
void AsmInterpreterCall::PopAsmInterpEntryFrame(ExtendedAssembler *assembler)
{
__ Addq(8, rsp);
Register fpRegister = r10;
__ Popq(fpRegister);
__ Addq(FRAME_SLOT_SIZE, rsp);
__ Popq(rbp);
__ PopAlignBytes();
__ Popq(rdi);
__ Movq(fpRegister, Operand(rdi, JSThread::GlueData::GetLeaveFrameOffset(false)));
size_t begin = __ GetCurrentPosition();
if (!assembler->FromInterpreterHandler()) {
__ PopCppCalleeSaveRegisters();
size_t end = __ GetCurrentPosition();
if ((end - begin) != FrameCompletionPos::X64AsmInterpToCpp) {
LOG_COMPILER(FATAL) << (end - begin) << " != " << FrameCompletionPos::X64AsmInterpToCpp
<< "This frame has been modified, and the offset AsmInterpToCp should be updated too.";
}
}
}
void AsmInterpreterCall::GetDeclaredNumArgsFromCallField(ExtendedAssembler *assembler, Register callFieldRegister,
Register declaredNumArgsRegister)
{
__ Movq(callFieldRegister, declaredNumArgsRegister);
__ Shrq(Method::NumArgsBits::START_BIT, declaredNumArgsRegister);
__ Andq(Method::NumArgsBits::Mask() >> Method::NumArgsBits::START_BIT, declaredNumArgsRegister);
}
void AsmInterpreterCall::GetNumVregsFromCallField(ExtendedAssembler *assembler, Register callFieldRegister,
Register numVregsRegister)
{
__ Movq(callFieldRegister, numVregsRegister);
__ Shrq(Method::NumVregsBits::START_BIT, numVregsRegister);
__ Andq(Method::NumVregsBits::Mask() >> Method::NumVregsBits::START_BIT, numVregsRegister);
}
void AsmInterpreterCall::JSCallCommonEntry(ExtendedAssembler *assembler,
JSCallMode mode, FrameTransitionType type)
{
Label stackOverflow;
Register glueRegister = __ GlueRegister();
__ UpdateReadBarrier(glueRegister);
Register fpRegister = __ AvailableRegister1();
Register callFieldRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
Register argcRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG0);
__ Movq(rsp, fpRegister);
Register declaredNumArgsRegister = __ AvailableRegister2();
GetDeclaredNumArgsFromCallField(assembler, callFieldRegister, declaredNumArgsRegister);
Label slowPathEntry;
Label fastPathEntry;
Label pushCallThis;
auto argc = kungfu::AssemblerModule::GetArgcFromJSCallMode(mode);
if (argc >= 0) {
__ Cmpq(argc, declaredNumArgsRegister);
} else {
__ Cmpq(argcRegister, declaredNumArgsRegister);
}
__ Jne(&slowPathEntry);
__ Bind(&fastPathEntry);
JSCallCommonFastPath(assembler, mode, &stackOverflow);
__ Bind(&pushCallThis);
PushCallThis(assembler, mode, &stackOverflow, type);
__ Bind(&slowPathEntry);
JSCallCommonSlowPath(assembler, mode, &fastPathEntry, &pushCallThis, &stackOverflow);
__ Bind(&stackOverflow);
if (kungfu::AssemblerModule::IsJumpToCallCommonEntry(mode)) {
__ Movq(fpRegister, rsp);
Register tempRegister = __ AvailableRegister1();
if (glueRegister != r13) {
__ Movq(glueRegister, r13);
}
Register acc = rsi;
__ Movq(JSTaggedValue::VALUE_EXCEPTION, acc);
Register methodRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::METHOD);
Register callTargetRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_TARGET);
__ Movq(callTargetRegister, tempRegister);
__ Movq(Operand(methodRegister, Method::NATIVE_POINTER_OR_BYTECODE_ARRAY_OFFSET), r12);
__ Movq(Operand(tempRegister, JSFunction::RAW_PROFILE_TYPE_INFO_OFFSET), r14);
__ Movq(Operand(r14, ProfileTypeInfoCell::VALUE_OFFSET), r14);
__ Movq(Operand(methodRegister, Method::CONSTANT_POOL_OFFSET), rbx);
__ Movq(kungfu::BytecodeStubCSigns::ID_ThrowStackOverflowException, tempRegister);
__ Movq(Operand(glueRegister, tempRegister, Times8, JSThread::GlueData::GetBCStubEntriesOffset(false)),
tempRegister);
__ Jmp(tempRegister);
} else {
[[maybe_unused]] TempRegisterScope scope(assembler);
Register temp = __ TempRegister();
ThrowStackOverflowExceptionAndReturn(assembler, glueRegister, fpRegister, temp);
}
}
void AsmInterpreterCall::PushCallThisRangeAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallThisRangeAndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_WITH_ARGV, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallRangeAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallRangeAndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_WITH_ARGV, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallNewAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallNewAndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_CONSTRUCTOR_WITH_ARGV, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushSuperCallAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushSuperCallAndDispatch));
JSCallCommonEntry(assembler, JSCallMode::SUPER_CALL_WITH_ARGV, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallArgs3AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallArgs3AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_ARG3, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallArgs2AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallArgs2AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_ARG2, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallArg1AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallArg1AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_ARG1, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallArg0AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallArg0AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_ARG0, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallThisArg0AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallThisArg0AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG0, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallThisArg1AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallThisArg1AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG1, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallThisArgs2AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallThisArgs2AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG2, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::PushCallThisArgs3AndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallThisArgs3AndDispatch));
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG3, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::JSCallCommonFastPath(ExtendedAssembler *assembler, JSCallMode mode, Label *stackOverflow)
{
Register glueRegister = __ GlueRegister();
Register arg0 = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG0);
Register arg1 = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
Label pushCallThis;
auto argc = kungfu::AssemblerModule::GetArgcFromJSCallMode(mode);
if (argc < 0) {
Register argcRegister = arg0;
Register argvRegister = arg1;
__ Cmpq(0, argcRegister);
__ Jbe(&pushCallThis);
{
[[maybe_unused]] TempRegisterScope scope(assembler);
Register opRegister = __ TempRegister();
Register op2Register = __ AvailableRegister2();
PushArgsWithArgvAndCheckStack(assembler, glueRegister, argcRegister, argvRegister, opRegister, op2Register,
stackOverflow);
}
__ Bind(&pushCallThis);
} else if (argc > 0) {
if (argc > 2) {
if (mode == JSCallMode::CALL_THIS_ARG3_WITH_RETURN) {
Register arg2 = __ CppJSCallAvailableRegister1();
__ Pushq(arg2);
} else {
Register arg2 = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG2);
__ Pushq(arg2);
}
}
if (argc > 1) {
__ Pushq(arg1);
}
if (argc > 0) {
__ Pushq(arg0);
}
}
}
void AsmInterpreterCall::JSCallCommonSlowPath(ExtendedAssembler *assembler, JSCallMode mode,
Label *fastPathEntry, Label *pushCallThis, Label *stackOverflow)
{
Register glueRegister = __ GlueRegister();
Register callFieldRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
Register argcRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG0);
Register arg0 = argcRegister;
Register arg1 = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
Label noExtraEntry;
Label pushArgsEntry;
auto argc = kungfu::AssemblerModule::GetArgcFromJSCallMode(mode);
Register declaredNumArgsRegister = __ AvailableRegister2();
__ Testq(Method::HaveExtraBit::Mask(), callFieldRegister);
__ Jz(&noExtraEntry);
{
[[maybe_unused]] TempRegisterScope scope(assembler);
Register tempArgcRegister = __ TempRegister();
if (argc >= 0) {
__ PushArgc(argc, tempArgcRegister);
} else {
__ PushArgc(argcRegister, tempArgcRegister);
}
}
__ Bind(&noExtraEntry);
{
if (argc == 0) {
Register op1 = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
[[maybe_unused]] TempRegisterScope scope(assembler);
Register op2 = __ TempRegister();
PushUndefinedWithArgcAndCheckStack(assembler, glueRegister, declaredNumArgsRegister, op1, op2,
stackOverflow);
__ Jmp(fastPathEntry);
return;
}
[[maybe_unused]] TempRegisterScope scope(assembler);
Register diffRegister = __ TempRegister();
__ Movq(declaredNumArgsRegister, diffRegister);
if (argc >= 0) {
__ Subq(argc, diffRegister);
} else {
__ Subq(argcRegister, diffRegister);
}
__ Cmpq(0, diffRegister);
__ Jle(&pushArgsEntry);
PushUndefinedWithArgc(assembler, diffRegister);
__ Jmp(fastPathEntry);
}
__ Bind(&pushArgsEntry);
__ Testq(Method::HaveExtraBit::Mask(), callFieldRegister);
__ Jnz(fastPathEntry);
if (argc == 1) {
__ Jmp(pushCallThis);
return;
}
__ Cmpq(0, declaredNumArgsRegister);
__ Je(pushCallThis);
if (argc < 0) {
Register argvRegister = arg1;
[[maybe_unused]] TempRegisterScope scope(assembler);
Register opRegister = __ TempRegister();
PushArgsWithArgvAndCheckStack(assembler, glueRegister, declaredNumArgsRegister, argvRegister, opRegister,
opRegister, stackOverflow);
} else if (argc > 0) {
Label pushArgs0;
if (argc > 2) {
__ Cmpq(1, declaredNumArgsRegister);
__ Je(&pushArgs0);
__ Pushq(arg1);
}
if (argc > 1) {
__ Bind(&pushArgs0);
__ Pushq(arg0);
}
}
__ Jmp(pushCallThis);
}
Register AsmInterpreterCall::GetThisRegsiter(ExtendedAssembler *assembler, JSCallMode mode, Register defaultRegister)
{
switch (mode) {
case JSCallMode::CALL_GETTER:
case JSCallMode::CALL_THIS_ARG0:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG0);
case JSCallMode::CALL_SETTER:
case JSCallMode::CALL_THIS_ARG1:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
case JSCallMode::CALL_THIS_ARG2:
case JSCallMode::CALL_CONSTRUCTOR_WITH_ARGV:
case JSCallMode::CALL_THIS_WITH_ARGV:
case JSCallMode::SUPER_CALL_WITH_ARGV:
case JSCallMode::SUPER_CALL_SPREAD_WITH_ARGV:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG2);
case JSCallMode::CALL_THIS_ARG3:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG3);
case JSCallMode::CALL_ENTRY:
case JSCallMode::CALL_FROM_AOT: {
Register argvRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
__ Movq(Operand(argvRegister, -FRAME_SLOT_SIZE), defaultRegister);
return defaultRegister;
}
case JSCallMode::CALL_THIS_ARG3_WITH_RETURN:
return __ CppJSCallAvailableRegister2();
case JSCallMode::CALL_THIS_ARG2_WITH_RETURN:
case JSCallMode::CALL_THIS_ARGV_WITH_RETURN: {
return __ CppJSCallAvailableRegister1();
}
default:
LOG_ECMA(FATAL) << "this branch is unreachable";
UNREACHABLE();
}
return invalidReg;
}
Register AsmInterpreterCall::GetNewTargetRegsiter(ExtendedAssembler *assembler, JSCallMode mode,
Register defaultRegister)
{
switch (mode) {
case JSCallMode::CALL_CONSTRUCTOR_WITH_ARGV:
case JSCallMode::CALL_THIS_WITH_ARGV:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_TARGET);
case JSCallMode::SUPER_CALL_WITH_ARGV:
case JSCallMode::SUPER_CALL_SPREAD_WITH_ARGV:
return __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG3);
case JSCallMode::CALL_FROM_AOT:
case JSCallMode::CALL_ENTRY: {
Register argvRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
__ Movq(Operand(argvRegister, -2 * FRAME_SLOT_SIZE), defaultRegister);
return defaultRegister;
}
default:
LOG_ECMA(FATAL) << "this branch is unreachable";
UNREACHABLE();
}
return invalidReg;
}
void AsmInterpreterCall::PushCallThis(ExtendedAssembler *assembler,
JSCallMode mode, Label *stackOverflow, FrameTransitionType type)
{
Register callFieldRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
Register callTargetRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_TARGET);
Register thisRegister = __ AvailableRegister2();
Label pushVregs;
Label pushNewTarget;
Label pushCallTarget;
bool haveThis = kungfu::AssemblerModule::JSModeHaveThisArg(mode);
bool haveNewTarget = kungfu::AssemblerModule::JSModeHaveNewTargetArg(mode);
if (!haveThis) {
__ Movq(JSTaggedValue::VALUE_UNDEFINED, thisRegister);
} else {
Register thisArgRegister = GetThisRegsiter(assembler, mode, thisRegister);
if (thisRegister != thisArgRegister) {
__ Movq(thisArgRegister, thisRegister);
}
}
__ Testb(CALL_TYPE_MASK, callFieldRegister);
__ Jz(&pushVregs);
__ Testq(Method::HaveThisBit::Mask(), callFieldRegister);
__ Jz(&pushNewTarget);
if (!haveThis) {
__ Pushq(JSTaggedValue::Undefined().GetRawData());
} else {
__ Pushq(thisRegister);
}
__ Bind(&pushNewTarget);
{
__ Testq(Method::HaveNewTargetBit::Mask(), callFieldRegister);
__ Jz(&pushCallTarget);
if (!haveNewTarget) {
__ Pushq(JSTaggedValue::Undefined().GetRawData());
} else {
[[maybe_unused]] TempRegisterScope scope(assembler);
Register defaultRegister = __ TempRegister();
Register newTargetRegister = GetNewTargetRegsiter(assembler, mode, defaultRegister);
__ Pushq(newTargetRegister);
}
}
__ Bind(&pushCallTarget);
{
__ Testq(Method::HaveFuncBit::Mask(), callFieldRegister);
__ Jz(&pushVregs);
__ Pushq(callTargetRegister);
}
__ Bind(&pushVregs);
{
PushVregs(assembler, stackOverflow, type);
}
}
void AsmInterpreterCall::PushVregs(ExtendedAssembler *assembler,
Label *stackOverflow, FrameTransitionType type)
{
Register glueRegister = __ GlueRegister();
Register prevSpRegister = rbp;
Register callTargetRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_TARGET);
Register methodRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::METHOD);
Register callFieldRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
Register fpRegister = __ AvailableRegister1();
Register thisRegister = __ AvailableRegister2();
Label pushFrameState;
[[maybe_unused]] TempRegisterScope scope(assembler);
Register tempRegister = __ TempRegister();
Register pcRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG0);
Register numVregsRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
GetNumVregsFromCallField(assembler, callFieldRegister, numVregsRegister);
__ Cmpq(0, numVregsRegister);
__ Jz(&pushFrameState);
Register temp2Register = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
PushUndefinedWithArgcAndCheckStack(assembler, glueRegister, numVregsRegister, tempRegister, temp2Register,
stackOverflow);
Register newSpRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::ARG1);
__ Bind(&pushFrameState);
{
StackOverflowCheck(assembler, glueRegister, numVregsRegister, tempRegister, temp2Register, stackOverflow);
__ Movq(rsp, newSpRegister);
PushFrameState(assembler, prevSpRegister, fpRegister,
callTargetRegister, thisRegister, methodRegister, pcRegister, tempRegister);
}
if (type == FrameTransitionType::OTHER_TO_BASELINE_CHECK ||
type == FrameTransitionType::BASELINE_TO_BASELINE_CHECK) {
__ Movq(Operand(callTargetRegister, JSFunction::BASELINECODE_OFFSET), tempRegister);
Label baselineCodeUndefined;
__ Cmpq(JSTaggedValue::Undefined().GetRawData(), tempRegister);
__ Je(&baselineCodeUndefined);
__ Cmpq(JSTaggedValue::Hole().GetRawData(), tempRegister);
__ Je(&baselineCodeUndefined);
Label stackAligned;
__ Testq(15, rsp);
__ Jz(&stackAligned);
__ PushAlignBytes();
__ Bind(&stackAligned);
__ Movq(Operand(tempRegister, MachineCode::FUNCADDR_OFFSET), tempRegister);
if (glueRegister != r13) {
__ Movq(glueRegister, r13);
}
if (methodRegister != rbx) {
__ Movq(methodRegister, rbx);
}
const int32_t pcOffsetFromSP = -24;
Register temp3Register = r10;
__ Movabs(std::numeric_limits<uint64_t>::max(), temp3Register);
__ Movq(temp3Register, Operand(newSpRegister, pcOffsetFromSP));
__ Movq(newSpRegister, rbp);
__ Jmp(tempRegister);
__ Bind(&baselineCodeUndefined);
}
DispatchCall(assembler, pcRegister, newSpRegister, callTargetRegister, methodRegister);
}
void AsmInterpreterCall::DispatchCall(ExtendedAssembler *assembler, Register pcRegister,
Register newSpRegister, Register callTargetRegister, Register methodRegister, Register accRegister,
bool hasException)
{
Register glueRegister = __ GlueRegister();
Label dispatchCall;
__ Testq(15, rsp);
__ Jnz(&dispatchCall);
__ PushAlignBytes();
__ Bind(&dispatchCall);
__ Movq(Operand(callTargetRegister, JSFunction::RAW_PROFILE_TYPE_INFO_OFFSET), r14);
__ Movq(Operand(r14, ProfileTypeInfoCell::VALUE_OFFSET), r14);
if (glueRegister != r13) {
__ Movq(glueRegister, r13);
}
__ Movq(newSpRegister, rbp);
__ Movzwq(Operand(methodRegister, Method::LITERAL_INFO_OFFSET), rdi);
__ Movq(Operand(methodRegister, Method::CONSTANT_POOL_OFFSET), rbx);
if (pcRegister != r12) {
__ Movq(pcRegister, r12);
}
Register bcIndexRegister = rax;
Register tempRegister = __ AvailableRegister1();
if (hasException) {
__ Movq(kungfu::BytecodeStubCSigns::ID_ExceptionHandler, bcIndexRegister);
} else {
__ Movzbq(Operand(pcRegister, 0), bcIndexRegister);
}
if (accRegister.IsValid()) {
ASSERT(accRegister == rsi);
} else {
__ Movq(JSTaggedValue::Hole().GetRawData(), rsi);
}
__ Movq(Operand(r13, bcIndexRegister, Times8, JSThread::GlueData::GetBCStubEntriesOffset(false)), tempRegister);
__ UpdateReadBarrier(r13);
__ Jmp(tempRegister);
}
void AsmInterpreterCall::PushCallRangeAndDispatchNative(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallRangeAndDispatchNative));
CallNativeWithArgv(assembler, false);
}
void AsmInterpreterCall::PushCallNewAndDispatchNative(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallNewAndDispatchNative));
CallNativeWithArgv(assembler, true);
}
void AsmInterpreterCall::PushNewTargetAndDispatchNative(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushNewTargetAndDispatchNative));
CallNativeWithArgv(assembler, true, true);
}
void AsmInterpreterCall::CallNativeWithArgv(ExtendedAssembler *assembler, bool callNew, bool hasNewTarget)
{
Register glue = rdi;
Register nativeCode = rsi;
Register func = rdx;
Register thisValue = rcx;
Register numArgs = r8;
Register stackArgs = r9;
Register temporary = rax;
Register temporary2 = r11;
Register opNumArgs = r10;
Label aligned;
Label pushThis;
Label stackOverflow;
bool isFrameComplete = PushBuiltinFrame(assembler, glue, FrameType::BUILTIN_FRAME_WITH_ARGV);
__ Push(numArgs);
__ Cmpq(0, numArgs);
__ Jz(&pushThis);
__ Movq(numArgs, opNumArgs);
PushArgsWithArgvAndCheckStack(assembler, glue, opNumArgs, stackArgs, temporary, temporary2, &stackOverflow);
__ Bind(&pushThis);
__ Push(thisValue);
if (callNew) {
if (hasNewTarget) {
Register newTarget = r12;
__ Movq(Operand(rsp, numArgs, Times8, 5 * FRAME_SLOT_SIZE), newTarget);
__ Pushq(newTarget);
} else {
__ Pushq(func);
}
} else {
__ Pushq(JSTaggedValue::Undefined().GetRawData());
}
__ Pushq(func);
if (!isFrameComplete) {
__ Leaq(Operand(rsp, numArgs, Times8, 5 * FRAME_SLOT_SIZE), rbp);
}
__ Movq(rsp, stackArgs);
__ Addl(NUM_MANDATORY_JSFUNC_ARGS, numArgs);
__ Pushq(numArgs);
#ifdef ENABLE_CMC_IR_FIX_REGISTER
Register calleeSaveGlue = r15;
__ Movq(glue, calleeSaveGlue);
#endif
__ Pushq(glue);
__ Movq(rsp, rdi);
__ Testq(0xf, rsp);
__ Jz(&aligned, Distance::Near);
__ PushAlignBytes();
__ Bind(&aligned);
CallNativeInternal(assembler, nativeCode);
__ Ret();
__ Bind(&stackOverflow);
{
Label aligneThrow;
__ Movq(Operand(glue, JSThread::GlueData::GetLeaveFrameOffset(false)), rsp);
__ Pushq(static_cast<int32_t>(FrameType::BUILTIN_FRAME_WITH_ARGV_STACK_OVER_FLOW_FRAME));
__ Pushq(0);
__ Pushq(JSTaggedValue::VALUE_UNDEFINED);
__ Pushq(JSTaggedValue::VALUE_UNDEFINED);
__ Pushq(JSTaggedValue::VALUE_UNDEFINED);
__ Leaq(Operand(rsp, 5 * FRAME_SLOT_SIZE), rbp);
__ Testq(0xf, rsp);
__ Jz(&aligneThrow, Distance::Near);
__ PushAlignBytes();
__ Bind(&aligneThrow);
Register trampolineIdRegister = r9;
Register trampolineRegister = r10;
__ Movq(kungfu::RuntimeStubCSigns::ID_ThrowStackOverflowException, trampolineIdRegister);
__ Movq(Operand(glue, trampolineIdRegister, Times8, JSThread::GlueData::GetRTStubEntriesOffset(false)),
trampolineRegister);
#ifdef ENABLE_CMC_IR_FIX_REGISTER
__ Movq(glue, r15);
#endif
__ Callq(trampolineRegister);
__ UpdateReadBarrier();
__ Movq(rbp, rsp);
__ Pop(rbp);
__ Ret();
}
}
void AsmInterpreterCall::CallNativeEntry(ExtendedAssembler *assembler, bool isJSFunction)
{
Label callFastBuiltin;
Label callNativeBuiltin;
Register glue = rdi;
Register argv = r9;
Register function = rsi;
Register nativeCode = r10;
if (isJSFunction) {
Register callFieldRegister = __ CallDispatcherArgument(kungfu::CallDispatchInputs::CALL_FIELD);
__ Movq(Operand(function, JSFunctionBase::CODE_ENTRY_OFFSET), nativeCode);
__ Btq(Method::IsFastBuiltinBit::START_BIT, callFieldRegister);
__ Jb(&callFastBuiltin);
} else {
Register method = rdx;
__ Movq(Operand(method, Method::NATIVE_POINTER_OR_BYTECODE_ARRAY_OFFSET), nativeCode);
}
__ Bind(&callNativeBuiltin);
if (isJSFunction) {
[[maybe_unused]] TempRegisterScope scope(assembler);
Register lexicalEnv = __ TempRegister();
Label next;
__ Movq(Operand(function, JSFunction::LEXICAL_ENV_OFFSET), lexicalEnv);
__ Cmpq(JSTaggedValue::Undefined().GetRawData(), lexicalEnv);
__ Je(&next);
__ Movq(lexicalEnv, Operand(glue, JSThread::GlueData::GetCurrentEnvOffset(false)));
__ Bind(&next);
}
__ PushAlignBytes();
__ Push(function);
__ Subq(3 * FRAME_SLOT_SIZE, rsp);
PushBuiltinFrame(assembler, glue, FrameType::BUILTIN_ENTRY_FRAME);
#ifdef ENABLE_CMC_IR_FIX_REGISTER
Register calleeSaveGlue = r15;
__ Movq(glue, calleeSaveGlue);
#endif
__ Movq(argv, r11);
__ Subq(2 * FRAME_SLOT_SIZE, r11);
__ Movq(r11, rdi);
CallNativeInternal(assembler, nativeCode);
__ Addq(5 * FRAME_SLOT_SIZE, rsp);
__ Ret();
__ Bind(&callFastBuiltin);
CallFastBuiltin(assembler, &callNativeBuiltin);
}
void AsmInterpreterCall::CallFastBuiltin(ExtendedAssembler *assembler, Label *callNativeBuiltin)
{
Label dispatchTable[3];
Label callEntryAndRet;
Register glue = rdi;
Register argc = r8;
Register argv = r9;
Register method = rdx;
Register function = rsi;
Register nativeCode = r10;
Register temp = rax;
Register temp1 = r11;
__ Movq(Operand(method, Method::EXTRA_LITERAL_INFO_OFFSET), temp1);
__ Shr(Method::BuiltinIdBits::START_BIT, temp1);
__ Andl((1LU << Method::BuiltinIdBits::SIZE) - 1, temp1);
__ Cmpl(static_cast<int32_t>(BUILTINS_STUB_ID(BUILTINS_CONSTRUCTOR_STUB_FIRST)), temp1);
__ Jge(callNativeBuiltin);
__ Cmpq(Immediate(3), argc);
__ Jg(callNativeBuiltin);
__ Movq(Operand(glue, temp1, Times8, JSThread::GlueData::GetBuiltinsStubEntriesOffset(false)), temp1);
PushAsmBridgeFrame(assembler);
__ Movq(function, temp);
__ Movq(nativeCode, rsi);
__ Movq(temp, rdx);
__ Movq(argv, temp);
__ Movq(argc, r9);
__ Movq(Operand(temp, FRAME_SLOT_SIZE), rcx);
__ Movq(Operand(temp, DOUBLE_SLOT_SIZE), r8);
__ Cmp(Immediate(0), r9);
__ Je(&dispatchTable[0]);
__ Cmp(Immediate(1), r9);
__ Je(&dispatchTable[1]);
__ Cmp(Immediate(2), r9);
__ Je(&dispatchTable[2]);
__ Movq(Operand(temp, QUINTUPLE_SLOT_SIZE), r10);
__ Pushq(r10);
__ Movq(Operand(temp, QUADRUPLE_SLOT_SIZE), r10);
__ Pushq(r10);
__ Movq(Operand(temp, TRIPLE_SLOT_SIZE), r10);
__ Pushq(r10);
__ Jmp(&callEntryAndRet);
__ Bind(&dispatchTable[0]);
{
__ Pushq(JSTaggedValue::VALUE_UNDEFINED);
__ Pushq(JSTaggedValue::VALUE_UNDEFINED);
__ Pushq(JSTaggedValue::VALUE_UNDEFINED);
__ Jmp(&callEntryAndRet);
}
__ Bind(&dispatchTable[1]);
{
__ Pushq(JSTaggedValue::VALUE_UNDEFINED);
__ Pushq(JSTaggedValue::VALUE_UNDEFINED);
__ Movq(Operand(temp, TRIPLE_SLOT_SIZE), r10);
__ Pushq(r10);
__ Jmp(&callEntryAndRet);
}
__ Bind(&dispatchTable[2]);
{
__ Pushq(JSTaggedValue::VALUE_UNDEFINED);
__ Movq(Operand(temp, QUADRUPLE_SLOT_SIZE), r10);
__ Pushq(r10);
__ Movq(Operand(temp, TRIPLE_SLOT_SIZE), r10);
__ Pushq(r10);
}
__ Bind(&callEntryAndRet);
{
__ Callq(temp1);
__ Addq(QUADRUPLE_SLOT_SIZE, rsp);
__ Pop(rbp);
__ Ret();
}
}
void AsmInterpreterCall::PushCallArgsAndDispatchNative(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(PushCallArgsAndDispatchNative));
Register nativeCode = rax;
Register glue = rdi;
__ Movq(Operand(rsp, FRAME_SLOT_SIZE), glue);
PushBuiltinFrame(assembler, glue, FrameType::BUILTIN_FRAME);
#ifdef ENABLE_CMC_IR_FIX_REGISTER
Register calleeSaveGlue = r15;
__ Movq(glue, calleeSaveGlue);
#endif
__ Leaq(Operand(rbp, 2 * FRAME_SLOT_SIZE), rdi);
__ PushAlignBytes();
CallNativeInternal(assembler, nativeCode);
__ Ret();
}
bool AsmInterpreterCall::PushBuiltinFrame(ExtendedAssembler *assembler,
Register glue, FrameType type)
{
__ Pushq(rbp);
__ Movq(rsp, Operand(glue, JSThread::GlueData::GetLeaveFrameOffset(false)));
__ Pushq(static_cast<int32_t>(type));
if (type != FrameType::BUILTIN_FRAME_WITH_ARGV) {
__ Leaq(Operand(rsp, FRAME_SLOT_SIZE), rbp);
return true;
} else if (type == FrameType::BUILTIN_FRAME_WITH_ARGV) {
return false;
} else {
LOG_ECMA(FATAL) << "this branch is unreachable";
UNREACHABLE();
}
}
void AsmInterpreterCall::CallNativeInternal(ExtendedAssembler *assembler, Register nativeCode)
{
__ Callq(nativeCode);
__ UpdateReadBarrier();
__ Movq(rbp, rsp);
__ Pop(rbp);
}
void AsmInterpreterCall::ResumeRspAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeRspAndDispatch));
Register glueRegister = __ GlueRegister();
Register spRegister = rbp;
Register pcRegister = r12;
Register ret = rsi;
Register jumpSizeRegister = r8;
Register frameStateBaseRegister = r11;
__ Movq(spRegister, frameStateBaseRegister);
__ Subq(AsmInterpretedFrame::GetSize(false), frameStateBaseRegister);
Label dispatch;
Label newObjectRangeReturn;
__ Cmpq(0, jumpSizeRegister);
__ Jle(&newObjectRangeReturn);
__ Movq(Operand(frameStateBaseRegister, AsmInterpretedFrame::GetBaseOffset(false)), spRegister);
__ Addq(jumpSizeRegister, pcRegister);
Register temp = rax;
Register opcodeRegister = rax;
__ Movzbq(Operand(pcRegister, 0), opcodeRegister);
__ Bind(&dispatch);
{
__ Movq(Operand(frameStateBaseRegister, AsmInterpretedFrame::GetFpOffset(false)), rsp);
Register bcStubRegister = r11;
__ Movq(Operand(glueRegister, opcodeRegister, Times8, JSThread::GlueData::GetBCStubEntriesOffset(false)),
bcStubRegister);
__ Jmp(bcStubRegister);
}
Label getThis;
Label notUndefined;
__ Bind(&newObjectRangeReturn);
__ Cmpq(JSTaggedValue::Undefined().GetRawData(), ret);
__ Jne(¬Undefined);
__ Bind(&getThis);
__ Movq(Operand(frameStateBaseRegister, AsmInterpretedFrame::GetBaseOffset(false)), spRegister);
__ Subq(jumpSizeRegister, pcRegister);
__ Movzbq(Operand(pcRegister, 0), opcodeRegister);
{
__ Movq(Operand(frameStateBaseRegister, AsmInterpretedFrame::GetThisOffset(false)), ret);
__ Movq(Operand(frameStateBaseRegister, AsmInterpretedFrame::GetFpOffset(false)), rsp);
Register bcStubRegister = r11;
__ Movq(Operand(glueRegister, opcodeRegister, Times8, JSThread::GlueData::GetBCStubEntriesOffset(false)),
bcStubRegister);
__ Jmp(bcStubRegister);
}
__ Bind(¬Undefined);
{
Label notEcmaObject;
__ Movabs(JSTaggedValue::TAG_HEAPOBJECT_MASK, temp);
__ And(ret, temp);
__ Cmpq(0, temp);
__ Jne(¬EcmaObject);
__ Movq(Operand(ret, JSFunction::HCLASS_OFFSET), temp);
Register maskRegister = r10;
__ Movabs(TaggedObject::GC_STATE_MASK, maskRegister);
__ And(maskRegister, temp);
__ Movl(Operand(temp, JSHClass::BIT_FIELD_OFFSET), temp);
__ Cmpb(static_cast<int32_t>(JSType::ECMA_OBJECT_LAST), temp);
__ Ja(¬EcmaObject);
__ Cmpb(static_cast<int32_t>(JSType::ECMA_OBJECT_FIRST), temp);
__ Jb(¬EcmaObject);
__ Movq(Operand(frameStateBaseRegister, AsmInterpretedFrame::GetBaseOffset(false)), spRegister);
__ Subq(jumpSizeRegister, pcRegister);
__ Movzbq(Operand(pcRegister, 0), opcodeRegister);
__ Jmp(&dispatch);
__ Bind(¬EcmaObject);
{
__ Movq(Operand(frameStateBaseRegister, AsmInterpretedFrame::GetFunctionOffset(false)), temp);
__ Movq(Operand(temp, JSFunctionBase::METHOD_OFFSET), temp);
__ Movq(Operand(temp, Method::EXTRA_LITERAL_INFO_OFFSET), temp);
__ Shr(Method::FunctionKindBits::START_BIT, temp);
__ Andl((1LU << Method::FunctionKindBits::SIZE) - 1, temp);
__ Cmpl(static_cast<int32_t>(FunctionKind::CLASS_CONSTRUCTOR), temp);
__ Jbe(&getThis);
}
{
__ Movq(Operand(frameStateBaseRegister, AsmInterpretedFrame::GetBaseOffset(false)), spRegister);
__ Movq(kungfu::BytecodeStubCSigns::ID_NewObjectRangeThrowException, opcodeRegister);
__ Jmp(&dispatch);
}
}
}
void AsmInterpreterCall::CallGetter(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(CallGetter));
Label target;
PushAsmInterpBridgeFrame(assembler);
__ Callq(&target);
PopAsmInterpBridgeFrame(assembler);
__ Ret();
__ Bind(&target);
JSCallCommonEntry(assembler, JSCallMode::CALL_GETTER, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::CallSetter(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(CallSetter));
Label target;
PushAsmInterpBridgeFrame(assembler);
__ Callq(&target);
PopAsmInterpBridgeFrame(assembler);
__ Ret();
__ Bind(&target);
JSCallCommonEntry(assembler, JSCallMode::CALL_SETTER, FrameTransitionType::OTHER_TO_OTHER);
}
void AsmInterpreterCall::CallReturnWithArgv(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(CallReturnWithArgv));
Label target;
PushAsmInterpBridgeFrame(assembler);
Register r13 = __ CppJSCallAvailableRegister1();
__ Movq(Operand(rbp, FRAME_SLOT_SIZE), r13);
__ Callq(&target);
PopAsmInterpBridgeFrame(assembler);
__ Ret();
__ Bind(&target);
{
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARGV_WITH_RETURN,
FrameTransitionType::OTHER_TO_OTHER);
}
}
void AsmInterpreterCall::CallContainersArgs2(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(CallContainersArgs2));
Label target;
PushAsmInterpBridgeFrame(assembler);
GetArgvAtStack(assembler);
__ Callq(&target);
PopAsmInterpBridgeFrame(assembler);
__ Ret();
__ Bind(&target);
{
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG2_WITH_RETURN,
FrameTransitionType::OTHER_TO_OTHER);
}
}
void AsmInterpreterCall::CallContainersArgs3(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(CallContainersArgs3));
Label target;
PushAsmInterpBridgeFrame(assembler);
GetArgvAtStack(assembler);
__ Callq(&target);
PopAsmInterpBridgeFrame(assembler);
__ Ret();
__ Bind(&target);
{
JSCallCommonEntry(assembler, JSCallMode::CALL_THIS_ARG3_WITH_RETURN,
FrameTransitionType::OTHER_TO_OTHER);
}
}
void AsmInterpreterCall::ResumeRspAndReturn(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeRspAndReturn));
Register currentSp = r12;
Register fpRegister = r10;
intptr_t offset = AsmInterpretedFrame::GetFpOffsetAsIntptr(false) -
AsmInterpretedFrame::GetSizeAsIntptr(false);
__ Movq(Operand(currentSp, static_cast<int32_t>(offset)), fpRegister);
__ Movq(fpRegister, rsp);
{
__ Movq(r13, rax);
__ Ret();
}
}
void AsmInterpreterCall::ResumeRspAndReturnBaseline(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeRspAndReturnBaseline));
Register currentSp = rbx;
Register fpRegister = r10;
intptr_t fpOffset = static_cast<intptr_t>(AsmInterpretedFrame::GetFpOffset(false)) -
static_cast<intptr_t>(AsmInterpretedFrame::GetSize(false));
__ Movq(Operand(currentSp, static_cast<int32_t>(fpOffset)), fpRegister);
__ Movq(fpRegister, rsp);
Register ret = rbp;
Register jumpSizeRegister = r14;
Label getThis;
Label notUndefined;
Label normalReturn;
Label newObjectRangeReturn;
__ Cmpq(0, jumpSizeRegister);
__ Jg(&normalReturn);
__ Bind(&newObjectRangeReturn);
{
__ Cmpq(JSTaggedValue::Undefined().GetRawData(), ret);
__ Jne(¬Undefined);
__ Bind(&getThis);
intptr_t thisOffset = static_cast<intptr_t>(AsmInterpretedFrame::GetThisOffset(false)) -
static_cast<intptr_t>(AsmInterpretedFrame::GetSize(false));
__ Movq(Operand(currentSp, static_cast<int32_t>(thisOffset)), ret);
__ Jmp(&normalReturn);
__ Bind(¬Undefined);
{
Register temp = rax;
Label notEcmaObject;
__ Movabs(JSTaggedValue::TAG_HEAPOBJECT_MASK, temp);
__ And(ret, temp);
__ Cmpq(0, temp);
__ Jne(¬EcmaObject);
__ Movq(Operand(ret, JSFunction::HCLASS_OFFSET), temp);
Register maskRegister = r11;
__ Movabs(TaggedObject::GC_STATE_MASK, maskRegister);
__ And(maskRegister, temp);
__ Movl(Operand(temp, JSHClass::BIT_FIELD_OFFSET), temp);
__ Cmpb(static_cast<int32_t>(JSType::ECMA_OBJECT_LAST), temp);
__ Ja(¬EcmaObject);
__ Cmpb(static_cast<int32_t>(JSType::ECMA_OBJECT_FIRST), temp);
__ Jb(¬EcmaObject);
__ Jmp(&normalReturn);
__ Bind(¬EcmaObject);
{
intptr_t funcOffset = AsmInterpretedFrame::GetFunctionOffsetAsIntptr(false) -
AsmInterpretedFrame::GetSizeAsIntptr(false);
__ Movq(Operand(currentSp, static_cast<int32_t>(funcOffset)), temp);
__ Movq(Operand(temp, JSFunctionBase::METHOD_OFFSET), temp);
__ Movq(Operand(temp, Method::EXTRA_LITERAL_INFO_OFFSET), temp);
__ Shr(Method::FunctionKindBits::START_BIT, temp);
__ Andl((1LU << Method::FunctionKindBits::SIZE) - 1, temp);
__ Cmpl(static_cast<int32_t>(FunctionKind::CLASS_CONSTRUCTOR), temp);
__ Jbe(&getThis);
}
}
}
__ Bind(&normalReturn);
__ Movq(ret, rax);
__ Ret();
}
void AsmInterpreterCall::ResumeCaughtFrameAndDispatch(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeCaughtFrameAndDispatch));
Register glueRegister = __ GlueRegister();
Register pcRegister = r12;
Label dispatch;
Register fpRegister = r11;
__ Movq(Operand(glueRegister, JSThread::GlueData::GetLastFpOffset(false)), fpRegister);
__ Cmpq(0, fpRegister);
__ Jz(&dispatch);
__ Movq(fpRegister, rsp);
__ Bind(&dispatch);
{
Register opcodeRegister = rax;
__ Movzbq(Operand(pcRegister, 0), opcodeRegister);
Register bcStubRegister = r11;
__ Movq(Operand(glueRegister, opcodeRegister, Times8, JSThread::GlueData::GetBCStubEntriesOffset(false)),
bcStubRegister);
__ Jmp(bcStubRegister);
}
}
void AsmInterpreterCall::ResumeUncaughtFrameAndReturn(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeUncaughtFrameAndReturn));
Register glueRegister = __ GlueRegister();
Register acc(r12);
Register cppRet(rax);
Label ret;
Register fpRegister = r11;
__ Movq(Operand(glueRegister, JSThread::GlueData::GetLastFpOffset(false)), fpRegister);
__ Cmpq(0, fpRegister);
__ Jz(&ret);
__ Movq(fpRegister, rsp);
__ Bind(&ret);
__ Movq(acc, cppRet);
__ Ret();
}
void AsmInterpreterCall::ResumeRspAndRollback(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(ResumeRspAndRollback));
Register glueRegister = __ GlueRegister();
Register spRegister = rbp;
Register pcRegister = r12;
Register ret = rsi;
Register jumpSizeRegister = r8;
Register frameStateBaseRegister = r11;
__ Movq(spRegister, frameStateBaseRegister);
__ Subq(AsmInterpretedFrame::GetSize(false), frameStateBaseRegister);
__ Movq(Operand(frameStateBaseRegister, AsmInterpretedFrame::GetBaseOffset(false)), spRegister);
__ Addq(jumpSizeRegister, pcRegister);
Register opcodeRegister = rax;
__ Movzbq(Operand(pcRegister, 0), opcodeRegister);
__ Movq(Operand(frameStateBaseRegister, AsmInterpretedFrame::GetFunctionOffset(false)), ret);
__ Movq(Operand(frameStateBaseRegister, AsmInterpretedFrame::GetFpOffset(false)), rsp);
Register bcStubRegister = r11;
__ Movq(Operand(glueRegister, opcodeRegister, Times8, JSThread::GlueData::GetBCStubEntriesOffset(false)),
bcStubRegister);
__ Jmp(bcStubRegister);
}
void AsmInterpreterCall::PreserveMostCall(ExtendedAssembler* assembler)
{
{
__ Pushq(rbp);
__ Pushq(static_cast<int64_t>(FrameType::OPTIMIZED_FRAME));
__ Leaq(Operand(rsp, FRAME_SLOT_SIZE), rbp);
}
int32_t PreserveRegisterIndex = 9;
__ Subq(PreserveRegisterIndex * FRAME_SLOT_SIZE, rsp);
__ Movq(rdi, Operand(rsp, FRAME_SLOT_SIZE * (--PreserveRegisterIndex)));
__ Movq(rsi, Operand(rsp, FRAME_SLOT_SIZE * (--PreserveRegisterIndex)));
__ Movq(rdx, Operand(rsp, FRAME_SLOT_SIZE * (--PreserveRegisterIndex)));
__ Movq(rcx, Operand(rsp, FRAME_SLOT_SIZE * (--PreserveRegisterIndex)));
__ Movq(r8, Operand(rsp, FRAME_SLOT_SIZE * (--PreserveRegisterIndex)));
__ Movq(r9, Operand(rsp, FRAME_SLOT_SIZE * (--PreserveRegisterIndex)));
__ Movq(r10, Operand(rsp, FRAME_SLOT_SIZE * (--PreserveRegisterIndex)));
__ Movq(rax, Operand(rsp, FRAME_SLOT_SIZE * (--PreserveRegisterIndex)));
__ Callq(r11);
__ Movq(Operand(rsp, FRAME_SLOT_SIZE * (PreserveRegisterIndex++)), rax);
__ Movq(Operand(rsp, FRAME_SLOT_SIZE * (PreserveRegisterIndex++)), r10);
__ Movq(Operand(rsp, FRAME_SLOT_SIZE * (PreserveRegisterIndex++)), r9);
__ Movq(Operand(rsp, FRAME_SLOT_SIZE * (PreserveRegisterIndex++)), r8);
__ Movq(Operand(rsp, FRAME_SLOT_SIZE * (PreserveRegisterIndex++)), rcx);
__ Movq(Operand(rsp, FRAME_SLOT_SIZE * (PreserveRegisterIndex++)), rdx);
__ Movq(Operand(rsp, FRAME_SLOT_SIZE * (PreserveRegisterIndex++)), rsi);
__ Movq(Operand(rsp, FRAME_SLOT_SIZE * (PreserveRegisterIndex++)), rdi);
{
__ Addq(PreserveRegisterIndex * FRAME_SLOT_SIZE + FRAME_SLOT_SIZE, rsp);
__ Popq(rbp);
__ Ret();
}
}
void AsmInterpreterCall::ASMFastWriteBarrier(ExtendedAssembler* assembler)
{
ASSERT(IN_YOUNG_SPACE < SHARED_SPACE_BEGIN && SHARED_SPACE_BEGIN <= SHARED_SWEEPABLE_SPACE_BEGIN &&
SHARED_SWEEPABLE_SPACE_END < IN_SHARED_READ_ONLY_SPACE && IN_SHARED_READ_ONLY_SPACE == HEAP_SPACE_END);
__ BindAssemblerStub(RTSTUB_ID(ASMFastWriteBarrier));
Label needCall;
Label checkMark;
Label needCallNotShare;
Label needShareBarrier;
Label valueNotShare;
Label valueMaybeSweepableShare;
{
__ Movabs(~(JSTaggedValue::TAG_MARK | DEFAULT_REGION_MASK), r11);
__ And(rcx, r11);
__ Movzbl(Operand(r11, 0), r11);
__ Cmpl(Immediate(RegionSpaceFlag::SHARED_SWEEPABLE_SPACE_BEGIN), r11);
__ Jae(&valueMaybeSweepableShare);
}
#if USE_STICKY_CMS_GC
__ Bind(&valueNotShare);
{
__ Movabs(~(JSTaggedValueInternals::TAG_WEAK), r11);
__ And(rcx, r11);
static_assert(TaggedStateWord::OLD_STATE == (static_cast<uint64_t>(ObjectState::OLD) <<
TaggedStateWord::ADDRESS_WIDTH));
static_assert(TaggedStateWord::YOUNG_STATE == (static_cast<uint64_t>(ObjectState::YOUNG) <<
TaggedStateWord::ADDRESS_WIDTH));
static_assert(TaggedStateWord::ADDRESS_WIDTH % GCBitset::BIT_PER_BYTE == 0);
__ Movzbl(Operand(r11, static_cast<uint32_t>(TaggedStateWord::ADDRESS_WIDTH) / GCBitset::BIT_PER_BYTE), r11);
__ Cmpl(Immediate(static_cast<uint32_t>(ObjectState::YOUNG)), r11);
__ Jne(&checkMark);
__ Movzbl(Operand(rsi, static_cast<uint32_t>(TaggedStateWord::ADDRESS_WIDTH) / GCBitset::BIT_PER_BYTE), r11);
__ Cmpl(Immediate(static_cast<uint32_t>(ObjectState::YOUNG)), r11);
__ Jne(&needCallNotShare);
}
#else
__ Bind(&valueNotShare);
{
__ Cmpl(Immediate(RegionSpaceFlag::IN_YOUNG_SPACE), r11);
__ Jne(&checkMark);
__ Movabs(~(JSTaggedValue::TAG_MARK | DEFAULT_REGION_MASK), r11);
__ And(rsi, r11);
__ Movzbl(Operand(r11, 0), r11);
__ Cmpl(Immediate(RegionSpaceFlag::IN_YOUNG_SPACE), r11);
__ Jne(&needCallNotShare);
}
#endif
__ Bind(&checkMark);
{
__ Movl(Operand(rdi, JSThread::GlueData::GetGCStateBitFieldOffset(false)), r11);
__ Testb(Immediate(JSThread::CONCURRENT_MARKING_BITFIELD_MASK), r11);
__ Jne(&needCallNotShare);
__ Ret();
}
__ Bind(&valueMaybeSweepableShare);
{
__ Cmpl(Immediate(RegionSpaceFlag::IN_SHARED_READ_ONLY_SPACE), r11);
__ Jne(&needShareBarrier);
__ Ret();
}
__ Bind(&needCallNotShare);
{
int32_t NonSValueBarrier = static_cast<int32_t>(JSThread::GlueData::GetCOStubEntriesOffset(false)) +
kungfu::CommonStubCSigns::SetNonSValueWithBarrier * FRAME_SLOT_SIZE;
__ Movq(Operand(rdi, NonSValueBarrier), r11);
}
__ Bind(&needCall);
{
PreserveMostCall(assembler);
}
__ Bind(&needShareBarrier);
{
ASMFastSharedWriteBarrier(assembler, needCall);
}
}
void AsmInterpreterCall::LoadBarrierCopyBack(ExtendedAssembler *assembler)
{
__ BindAssemblerStub(RTSTUB_ID(LoadBarrierCopyBack));
return;
}
void AsmInterpreterCall::ASMFastSharedWriteBarrier(ExtendedAssembler* assembler, Label& needcall)
{
Label checkBarrierForSharedValue;
Label restoreScratchRegister;
Label callSharedBarrier;
{
__ Movabs(~(JSTaggedValue::TAG_MARK | DEFAULT_REGION_MASK), r11);
__ And(rsi, r11);
__ Movzbl(Operand(r11, 0), r11);
__ Cmpl(Immediate(RegionSpaceFlag::SHARED_SPACE_BEGIN), r11);
__ Jae(&checkBarrierForSharedValue);
}
{
__ Movabs(~(JSTaggedValue::TAG_MARK | DEFAULT_REGION_MASK), r11);
__ And(rsi, r11);
__ Movq(Operand(r11, Region::PackedData::GetLocalToShareSetOffset(false)), r11);
__ Cmpq(Immediate(0), r11);
__ Je(&callSharedBarrier);
}
{
{
__ Pushq(r12);
__ Pushq(r13);
}
__ Movabs(DEFAULT_REGION_MASK, r12);
__ And(rsi, r12);
__ Addq(rdx, r12);
__ Movzbl(r12, r13);
constexpr uint32_t byteIndexMask = static_cast<uint32_t>(0xffffffffffffffff >> TAGGED_TYPE_SIZE_LOG) >>
GCBitset::BIT_PER_WORD_LOG2 << GCBitset::BYTE_PER_WORD_LOG2;
static_assert(byteIndexMask == 0x1ffffffc && "LocalToShareSet is changed?");
__ Shrq(TAGGED_TYPE_SIZE_LOG + GCBitset::BIT_PER_WORD_LOG2 - GCBitset::BYTE_PER_WORD_LOG2, r12);
__ Andq(byteIndexMask, r12);
__ Addq(RememberedSet::GCBITSET_DATA_OFFSET, r11);
__ Addq(r12, r11);
__ Movl(Operand(r11, 0), r12);
__ Shrl(TAGGED_TYPE_SIZE_LOG, r13);
__ Btsl(r13, r12);
__ Jb(&restoreScratchRegister);
__ Movl(r12, Operand(r11, 0));
}
__ Bind(&restoreScratchRegister);
{
__ Popq(r13);
__ Popq(r12);
}
__ Bind(&checkBarrierForSharedValue);
{
__ Movl(Operand(rdi, JSThread::GlueData::GetSharedGCStateBitFieldOffset(false)), r11);
__ Testb(Immediate(JSThread::SHARED_CONCURRENT_MARKING_BITFIELD_MASK), r11);
__ Jne(&callSharedBarrier);
__ Ret();
}
__ Bind(&callSharedBarrier);
{
int32_t NonSValueBarrier = static_cast<int32_t>(JSThread::GlueData::GetCOStubEntriesOffset(false)) +
kungfu::CommonStubCSigns::SetSValueWithBarrier * FRAME_SLOT_SIZE;
__ Movq(Operand(rdi, NonSValueBarrier), r11);
__ Jmp(&needcall);
}
}
void AsmInterpreterCall::PushUndefinedWithArgcAndCheckStack(ExtendedAssembler *assembler, Register glue, Register argc,
Register op1, Register op2, Label *stackOverflow)
{
ASSERT(stackOverflow != nullptr);
StackOverflowCheck(assembler, glue, argc, op1, op2, stackOverflow);
PushUndefinedWithArgc(assembler, argc);
}
void AsmInterpreterCall::ThrowStackOverflowExceptionAndReturn(ExtendedAssembler *assembler, Register glue,
Register fpReg, Register op)
{
if (fpReg != rsp) {
__ Movq(fpReg, rsp);
}
__ Movq(kungfu::RuntimeStubCSigns::ID_ThrowStackOverflowException, op);
__ Movq(Operand(glue, op, Times8, JSThread::GlueData::GetRTStubEntriesOffset(false)), op);
if (glue != r13) {
__ Movq(glue, r13);
}
__ Pushq(rbp);
__ Pushq(static_cast<int64_t>(FrameType::ASM_BRIDGE_FRAME));
__ Leaq(Operand(rsp, FRAME_SLOT_SIZE), rbp);
Label callRuntime;
__ Testq(0x8, rsp);
__ Jnz(&callRuntime);
__ PushAlignBytes();
__ Bind(&callRuntime);
__ Pushq(r10);
__ Pushq(0);
__ Pushq(kungfu::RuntimeStubCSigns::ID_ThrowStackOverflowException);
__ Movq(glue, rax);
__ Movq(kungfu::RuntimeStubCSigns::ID_CallRuntime, r10);
__ Movq(Operand(rax, r10, Times8, JSThread::GlueData::GetRTStubEntriesOffset(false)), r10);
__ Callq(r10);
__ Addq(2 * FRAME_SLOT_SIZE, rsp);
__ Popq(r10);
__ Movq(rbp, rsp);
__ Popq(rbp);
__ Ret();
}
void AsmInterpreterCall::ThrowStackOverflowExceptionAndReturnToAsmInterpBridgeFrame(ExtendedAssembler *assembler,
Register glue, Register fpReg, Register op)
{
if (fpReg != rsp) {
__ Movq(fpReg, rsp);
}
__ Movq(kungfu::RuntimeStubCSigns::ID_ThrowStackOverflowException, op);
__ Movq(Operand(glue, op, Times8, JSThread::GlueData::GetRTStubEntriesOffset(false)), op);
if (glue != r13) {
__ Movq(glue, r13);
}
__ Pushq(rbp);
__ Pushq(static_cast<int64_t>(FrameType::ASM_BRIDGE_FRAME));
__ Leaq(Operand(rsp, FRAME_SLOT_SIZE), rbp);
Label callRuntime;
__ Testq(0x8, rsp);
__ Jnz(&callRuntime);
__ PushAlignBytes();
__ Bind(&callRuntime);
__ Pushq(r10);
__ Pushq(0);
__ Pushq(kungfu::RuntimeStubCSigns::ID_ThrowStackOverflowException);
__ Movq(glue, rax);
__ Movq(kungfu::RuntimeStubCSigns::ID_CallRuntime, r10);
__ Movq(Operand(rax, r10, Times8, JSThread::GlueData::GetRTStubEntriesOffset(false)), r10);
__ Callq(r10);
__ Addq(2 * FRAME_SLOT_SIZE, rsp);
__ Popq(r10);
__ Movq(rbp, rsp);
__ Popq(rbp);
int32_t skipNum = static_cast<int32_t>(AsmInterpretedBridgeFrame::GetSize(false)) / FRAME_SLOT_SIZE + 5 + 1;
__ Leaq(Operand(rbp, -skipNum * FRAME_SLOT_SIZE), rsp);
__ Ret();
}
void AsmInterpreterCall::HasPendingException([[maybe_unused]] ExtendedAssembler *assembler,
[[maybe_unused]] Register threadRegister)
{
}
#undef __
}