#include <math.h>
#include <stdint.h>
#include <stdlib.h>
#include <limits>
#include "src/base/bits.h"
#include "src/base/float16.h"
#include "src/base/ieee754.h"
#include "src/base/numerics/safe_conversions.h"
#include "src/common/assert-scope.h"
#include "src/execution/frames-inl.h"
#include "src/execution/frames.h"
#include "src/execution/pointer-authentication.h"
#include "src/numbers/conversions.h"
#include "src/numbers/ieee754.h"
#include "src/roots/roots-inl.h"
#include "src/utils/memcopy.h"
#include "src/wasm/wasm-engine.h"
#include "src/wasm/wasm-objects-inl.h"
#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \
defined(THREAD_SANITIZER) || defined(LEAK_SANITIZER) || \
defined(UNDEFINED_SANITIZER)
#define V8_WITH_SANITIZER
#endif
#if defined(V8_OS_WIN) && defined(V8_WITH_SANITIZER)
#define RESET_THREAD_IN_WASM_FLAG_FOR_ASAN_ON_WINDOWS
#include "src/trap-handler/trap-handler.h"
#endif
#include "src/base/memory.h"
#include "src/base/overflowing-math.h"
#include "src/utils/utils.h"
#include "src/wasm/wasm-external-refs.h"
namespace v8::internal::wasm {
using base::ReadUnalignedValue;
using base::WriteUnalignedValue;
void f32_trunc_wrapper(Address data) {
WriteUnalignedValue<float>(data, truncf(ReadUnalignedValue<float>(data)));
}
void f32_floor_wrapper(Address data) {
WriteUnalignedValue<float>(data, floorf(ReadUnalignedValue<float>(data)));
}
void f32_ceil_wrapper(Address data) {
WriteUnalignedValue<float>(data, ceilf(ReadUnalignedValue<float>(data)));
}
void f32_nearest_int_wrapper(Address data) {
float input = ReadUnalignedValue<float>(data);
float value = nearbyintf(input);
#if V8_OS_AIX
value = FpOpWorkaround<float>(input, value);
#endif
WriteUnalignedValue<float>(data, value);
}
void f64_trunc_wrapper(Address data) {
WriteUnalignedValue<double>(data, trunc(ReadUnalignedValue<double>(data)));
}
void f64_floor_wrapper(Address data) {
WriteUnalignedValue<double>(data, floor(ReadUnalignedValue<double>(data)));
}
void f64_ceil_wrapper(Address data) {
WriteUnalignedValue<double>(data, ceil(ReadUnalignedValue<double>(data)));
}
void f64_nearest_int_wrapper(Address data) {
double input = ReadUnalignedValue<double>(data);
double value = nearbyint(input);
#if V8_OS_AIX
value = FpOpWorkaround<double>(input, value);
#endif
WriteUnalignedValue<double>(data, value);
}
void int64_to_float32_wrapper(Address data) {
int64_t input = ReadUnalignedValue<int64_t>(data);
WriteUnalignedValue<float>(data, static_cast<float>(input));
}
void uint64_to_float32_wrapper(Address data) {
uint64_t input = ReadUnalignedValue<uint64_t>(data);
#if defined(V8_OS_WIN)
// 00000000000001XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX100000000000000
uint32_t leading_zeros = base::bits::CountLeadingZeros(input);
uint32_t trailing_zeros = base::bits::CountTrailingZeros(input);
constexpr uint32_t num_extracted_bits = 25;
if (leading_zeros + trailing_zeros + num_extracted_bits < 64) {
uint32_t num_aggregation_bits = 64 - num_extracted_bits - leading_zeros;
int32_t extracted_bits =
static_cast<int32_t>(input >> (num_aggregation_bits - 1));
extracted_bits |= 1;
float result = static_cast<float>(extracted_bits);
int32_t shift_back = static_cast<int32_t>(num_aggregation_bits) - 1 - 2;
int32_t multiplier_bits = ((shift_back - 127) & 0xff) << 23;
result *= base::bit_cast<float>(multiplier_bits);
WriteUnalignedValue<float>(data, result);
return;
}
#endif
WriteUnalignedValue<float>(data, static_cast<float>(input));
}
void int64_to_float64_wrapper(Address data) {
int64_t input = ReadUnalignedValue<int64_t>(data);
WriteUnalignedValue<double>(data, static_cast<double>(input));
}
void uint64_to_float64_wrapper(Address data) {
uint64_t input = ReadUnalignedValue<uint64_t>(data);
double result = static_cast<double>(input);
#if V8_CC_MSVC
uint32_t low_word = static_cast<uint32_t>(input & 0xFFFFFFFF);
uint32_t high_word = static_cast<uint32_t>(input >> 32);
double shift = static_cast<double>(1ull << 32);
result = static_cast<double>(high_word);
result *= shift;
result += static_cast<double>(low_word);
#endif
WriteUnalignedValue<double>(data, result);
}
int32_t float32_to_int64_wrapper(Address data) {
float input = ReadUnalignedValue<float>(data);
if (base::IsValueInRangeForNumericType<int64_t>(input)) {
WriteUnalignedValue<int64_t>(data, static_cast<int64_t>(input));
return 1;
}
return 0;
}
int32_t float32_to_uint64_wrapper(Address data) {
float input = ReadUnalignedValue<float>(data);
if (base::IsValueInRangeForNumericType<uint64_t>(input)) {
WriteUnalignedValue<uint64_t>(data, static_cast<uint64_t>(input));
return 1;
}
return 0;
}
int32_t float64_to_int64_wrapper(Address data) {
double input = ReadUnalignedValue<double>(data);
if (base::IsValueInRangeForNumericType<int64_t>(input)) {
WriteUnalignedValue<int64_t>(data, static_cast<int64_t>(input));
return 1;
}
return 0;
}
int32_t float64_to_uint64_wrapper(Address data) {
double input = ReadUnalignedValue<double>(data);
if (base::IsValueInRangeForNumericType<uint64_t>(input)) {
WriteUnalignedValue<uint64_t>(data, static_cast<uint64_t>(input));
return 1;
}
return 0;
}
void float32_to_int64_sat_wrapper(Address data) {
float input = ReadUnalignedValue<float>(data);
if (base::IsValueInRangeForNumericType<int64_t>(input)) {
WriteUnalignedValue<int64_t>(data, static_cast<int64_t>(input));
return;
}
if (std::isnan(input)) {
WriteUnalignedValue<int64_t>(data, 0);
return;
}
if (input < 0.0) {
WriteUnalignedValue<int64_t>(data, std::numeric_limits<int64_t>::min());
return;
}
WriteUnalignedValue<int64_t>(data, std::numeric_limits<int64_t>::max());
}
void float32_to_uint64_sat_wrapper(Address data) {
float input = ReadUnalignedValue<float>(data);
if (base::IsValueInRangeForNumericType<uint64_t>(input)) {
WriteUnalignedValue<uint64_t>(data, static_cast<uint64_t>(input));
return;
}
if (input >= static_cast<float>(std::numeric_limits<uint64_t>::max())) {
WriteUnalignedValue<uint64_t>(data, std::numeric_limits<uint64_t>::max());
return;
}
WriteUnalignedValue<uint64_t>(data, 0);
}
void float64_to_int64_sat_wrapper(Address data) {
double input = ReadUnalignedValue<double>(data);
if (base::IsValueInRangeForNumericType<int64_t>(input)) {
WriteUnalignedValue<int64_t>(data, static_cast<int64_t>(input));
return;
}
if (std::isnan(input)) {
WriteUnalignedValue<int64_t>(data, 0);
return;
}
if (input < 0.0) {
WriteUnalignedValue<int64_t>(data, std::numeric_limits<int64_t>::min());
return;
}
WriteUnalignedValue<int64_t>(data, std::numeric_limits<int64_t>::max());
}
void float64_to_uint64_sat_wrapper(Address data) {
double input = ReadUnalignedValue<double>(data);
if (base::IsValueInRangeForNumericType<uint64_t>(input)) {
WriteUnalignedValue<uint64_t>(data, static_cast<uint64_t>(input));
return;
}
if (input >= static_cast<double>(std::numeric_limits<uint64_t>::max())) {
WriteUnalignedValue<uint64_t>(data, std::numeric_limits<uint64_t>::max());
return;
}
WriteUnalignedValue<uint64_t>(data, 0);
}
void float16_to_float32_wrapper(Address data) {
WriteUnalignedValue<float>(data, Float16::Read(data).ToFloat32());
}
void float32_to_float16_wrapper(Address data) {
Float16::FromFloat32(ReadUnalignedValue<float>(data)).Write(data);
}
int32_t int64_div_wrapper(Address data) {
int64_t dividend = ReadUnalignedValue<int64_t>(data);
int64_t divisor = ReadUnalignedValue<int64_t>(data + sizeof(dividend));
if (divisor == 0) {
return 0;
}
if (divisor == -1 && dividend == std::numeric_limits<int64_t>::min()) {
return -1;
}
WriteUnalignedValue<int64_t>(data, dividend / divisor);
return 1;
}
int32_t int64_mod_wrapper(Address data) {
int64_t dividend = ReadUnalignedValue<int64_t>(data);
int64_t divisor = ReadUnalignedValue<int64_t>(data + sizeof(dividend));
if (divisor == 0) {
return 0;
}
if (divisor == -1 && dividend == std::numeric_limits<int64_t>::min()) {
WriteUnalignedValue<int64_t>(data, 0);
return 1;
}
WriteUnalignedValue<int64_t>(data, dividend % divisor);
return 1;
}
int32_t uint64_div_wrapper(Address data) {
uint64_t dividend = ReadUnalignedValue<uint64_t>(data);
uint64_t divisor = ReadUnalignedValue<uint64_t>(data + sizeof(dividend));
if (divisor == 0) {
return 0;
}
WriteUnalignedValue<uint64_t>(data, dividend / divisor);
return 1;
}
int32_t uint64_mod_wrapper(Address data) {
uint64_t dividend = ReadUnalignedValue<uint64_t>(data);
uint64_t divisor = ReadUnalignedValue<uint64_t>(data + sizeof(dividend));
if (divisor == 0) {
return 0;
}
WriteUnalignedValue<uint64_t>(data, dividend % divisor);
return 1;
}
uint32_t word32_rol_wrapper(uint32_t input, uint32_t shift) {
return (input << (shift & 31)) | (input >> ((32 - shift) & 31));
}
uint32_t word32_ror_wrapper(uint32_t input, uint32_t shift) {
return (input >> (shift & 31)) | (input << ((32 - shift) & 31));
}
uint64_t word64_rol_wrapper(uint64_t input, uint32_t shift) {
return (input << (shift & 63)) | (input >> ((64 - shift) & 63));
}
uint64_t word64_ror_wrapper(uint64_t input, uint32_t shift) {
return (input >> (shift & 63)) | (input << ((64 - shift) & 63));
}
void float64_pow_wrapper(Address data) {
double x = ReadUnalignedValue<double>(data);
double y = ReadUnalignedValue<double>(data + sizeof(x));
WriteUnalignedValue<double>(data, math::pow(x, y));
}
template <typename T, T (*float_round_op)(T)>
void simd_float_round_wrapper(Address data) {
constexpr int n = kSimd128Size / sizeof(T);
for (int i = 0; i < n; i++) {
T input = ReadUnalignedValue<T>(data + (i * sizeof(T)));
T value = float_round_op(input);
#if V8_OS_AIX
value = FpOpWorkaround<T>(input, value);
#endif
WriteUnalignedValue<T>(data + (i * sizeof(T)), value);
}
}
void f64x2_ceil_wrapper(Address data) {
simd_float_round_wrapper<double, &ceil>(data);
}
void f64x2_floor_wrapper(Address data) {
simd_float_round_wrapper<double, &floor>(data);
}
void f64x2_trunc_wrapper(Address data) {
simd_float_round_wrapper<double, &trunc>(data);
}
void f64x2_nearest_int_wrapper(Address data) {
simd_float_round_wrapper<double, &nearbyint>(data);
}
void f32x4_ceil_wrapper(Address data) {
simd_float_round_wrapper<float, &ceilf>(data);
}
void f32x4_floor_wrapper(Address data) {
simd_float_round_wrapper<float, &floorf>(data);
}
void f32x4_trunc_wrapper(Address data) {
simd_float_round_wrapper<float, &truncf>(data);
}
void f32x4_nearest_int_wrapper(Address data) {
simd_float_round_wrapper<float, &nearbyintf>(data);
}
Float16 f16_abs(Float16 a) {
return Float16::FromFloat32(std::abs(a.ToFloat32()));
}
void f16x8_abs_wrapper(Address data) {
simd_float_round_wrapper<Float16, &f16_abs>(data);
}
Float16 f16_neg(Float16 a) { return Float16::FromFloat32(-(a.ToFloat32())); }
void f16x8_neg_wrapper(Address data) {
simd_float_round_wrapper<Float16, &f16_neg>(data);
}
Float16 f16_sqrt(Float16 a) {
return Float16::FromFloat32(std::sqrt(a.ToFloat32()));
}
void f16x8_sqrt_wrapper(Address data) {
simd_float_round_wrapper<Float16, &f16_sqrt>(data);
}
Float16 f16_ceil(Float16 a) {
return Float16::FromFloat32(ceilf(a.ToFloat32()));
}
void f16x8_ceil_wrapper(Address data) {
simd_float_round_wrapper<Float16, &f16_ceil>(data);
}
Float16 f16_floor(Float16 a) {
return Float16::FromFloat32(floorf(a.ToFloat32()));
}
void f16x8_floor_wrapper(Address data) {
simd_float_round_wrapper<Float16, &f16_floor>(data);
}
Float16 f16_trunc(Float16 a) {
return Float16::FromFloat32(truncf(a.ToFloat32()));
}
void f16x8_trunc_wrapper(Address data) {
simd_float_round_wrapper<Float16, &f16_trunc>(data);
}
Float16 f16_nearest_int(Float16 a) {
return Float16::FromFloat32(nearbyintf(a.ToFloat32()));
}
void f16x8_nearest_int_wrapper(Address data) {
simd_float_round_wrapper<Float16, &f16_nearest_int>(data);
}
template <typename R, R (*float_bin_op)(Float16, Float16)>
void simd_float16_bin_wrapper(Address data) {
constexpr int n = kSimd128Size / sizeof(Float16);
for (int i = 0; i < n; i++) {
Float16 lhs = Float16::Read(data + (i * sizeof(Float16)));
Float16 rhs = Float16::Read(data + kSimd128Size + (i * sizeof(Float16)));
R value = float_bin_op(lhs, rhs);
WriteUnalignedValue<R>(data + (i * sizeof(R)), value);
}
}
int16_t f16_eq(Float16 a, Float16 b) {
return a.ToFloat32() == b.ToFloat32() ? -1 : 0;
}
void f16x8_eq_wrapper(Address data) {
simd_float16_bin_wrapper<int16_t, &f16_eq>(data);
}
int16_t f16_ne(Float16 a, Float16 b) {
return a.ToFloat32() != b.ToFloat32() ? -1 : 0;
}
void f16x8_ne_wrapper(Address data) {
simd_float16_bin_wrapper<int16_t, &f16_ne>(data);
}
int16_t f16_lt(Float16 a, Float16 b) {
return a.ToFloat32() < b.ToFloat32() ? -1 : 0;
}
void f16x8_lt_wrapper(Address data) {
simd_float16_bin_wrapper<int16_t, &f16_lt>(data);
}
int16_t f16_le(Float16 a, Float16 b) {
return a.ToFloat32() <= b.ToFloat32() ? -1 : 0;
}
void f16x8_le_wrapper(Address data) {
simd_float16_bin_wrapper<int16_t, &f16_le>(data);
}
Float16 f16_add(Float16 a, Float16 b) {
return Float16::FromFloat32(a.ToFloat32() + b.ToFloat32());
}
void f16x8_add_wrapper(Address data) {
simd_float16_bin_wrapper<Float16, &f16_add>(data);
}
Float16 f16_sub(Float16 a, Float16 b) {
return Float16::FromFloat32(a.ToFloat32() - b.ToFloat32());
}
void f16x8_sub_wrapper(Address data) {
simd_float16_bin_wrapper<Float16, &f16_sub>(data);
}
Float16 f16_mul(Float16 a, Float16 b) {
return Float16::FromFloat32(a.ToFloat32() * b.ToFloat32());
}
void f16x8_mul_wrapper(Address data) {
simd_float16_bin_wrapper<Float16, &f16_mul>(data);
}
Float16 f16_div(Float16 a, Float16 b) {
return Float16::FromFloat32(base::Divide(a.ToFloat32(), b.ToFloat32()));
}
void f16x8_div_wrapper(Address data) {
simd_float16_bin_wrapper<Float16, &f16_div>(data);
}
Float16 f16_min(Float16 a, Float16 b) {
return Float16::FromFloat32(JSMin(a.ToFloat32(), b.ToFloat32()));
}
void f16x8_min_wrapper(Address data) {
simd_float16_bin_wrapper<Float16, &f16_min>(data);
}
Float16 f16_max(Float16 a, Float16 b) {
return Float16::FromFloat32(JSMax(a.ToFloat32(), b.ToFloat32()));
}
void f16x8_max_wrapper(Address data) {
simd_float16_bin_wrapper<Float16, &f16_max>(data);
}
Float16 f16_pmin(Float16 a, Float16 b) {
return Float16::FromFloat32(std::min(a.ToFloat32(), b.ToFloat32()));
}
void f16x8_pmin_wrapper(Address data) {
simd_float16_bin_wrapper<Float16, &f16_pmin>(data);
}
Float16 f16_pmax(Float16 a, Float16 b) {
return Float16::FromFloat32(std::max(a.ToFloat32(), b.ToFloat32()));
}
void f16x8_pmax_wrapper(Address data) {
simd_float16_bin_wrapper<Float16, &f16_pmax>(data);
}
template <typename T, typename R, R (*float_un_op)(T)>
void simd_float_un_wrapper(Address data) {
constexpr int n = kSimd128Size / sizeof(T);
for (int i = 0; i < n; i++) {
T input = ReadUnalignedValue<T>(data + (i * sizeof(T)));
R value = float_un_op(input);
WriteUnalignedValue<R>(data + (i * sizeof(T)), value);
}
}
int16_t ConvertToIntS(Float16 val) {
float f32 = val.ToFloat32();
if (std::isnan(f32)) return 0;
if (f32 > float{kMaxInt16}) return kMaxInt16;
if (f32 < float{kMinInt16}) return kMinInt16;
return static_cast<int16_t>(f32);
}
uint16_t ConvertToIntU(Float16 val) {
float f32 = val.ToFloat32();
if (std::isnan(f32)) return 0;
if (f32 > float{kMaxUInt16}) return kMaxUInt16;
if (f32 < float{0}) return 0;
return static_cast<uint16_t>(f32);
}
void i16x8_sconvert_f16x8_wrapper(Address data) {
simd_float_un_wrapper<Float16, int16_t, &ConvertToIntS>(data);
}
void i16x8_uconvert_f16x8_wrapper(Address data) {
simd_float_un_wrapper<Float16, uint16_t, &ConvertToIntU>(data);
}
Float16 ConvertToF16S(int16_t val) { return Float16::FromFloat32(val); }
void f16x8_sconvert_i16x8_wrapper(Address data) {
simd_float_un_wrapper<int16_t, Float16, &ConvertToF16S>(data);
}
Float16 ConvertToF16U(uint16_t val) { return Float16::FromFloat32(val); }
void f16x8_uconvert_i16x8_wrapper(Address data) {
simd_float_un_wrapper<uint16_t, Float16, &ConvertToF16U>(data);
}
void f32x4_promote_low_f16x8_wrapper(Address data) {
Float16 a = Float16::Read(data);
Float16 b = Float16::Read(data + sizeof(Float16));
Float16 c = Float16::Read(data + 2 * sizeof(Float16));
Float16 d = Float16::Read(data + 3 * sizeof(Float16));
WriteUnalignedValue<float>(data, a.ToFloat32());
WriteUnalignedValue<float>(data + sizeof(float), b.ToFloat32());
WriteUnalignedValue<float>(data + (2 * sizeof(float)), c.ToFloat32());
WriteUnalignedValue<float>(data + (3 * sizeof(float)), d.ToFloat32());
}
void f16x8_demote_f32x4_zero_wrapper(Address data) {
#if V8_TARGET_BIG_ENDIAN
for (int i = 3, j = 7; i >= 0; i--, j--) {
float input = ReadUnalignedValue<float>(data + (i * sizeof(float)));
Float16::FromFloat32(input).Write(data + (j * sizeof(Float16)));
}
for (int i = 0; i < 4; i++) {
WriteUnalignedValue<Float16>(data + (i * sizeof(Float16)),
Float16::FromFloat32(0));
}
#else
for (int i = 0; i < 4; i++) {
float input = ReadUnalignedValue<float>(data + (i * sizeof(float)));
Float16::FromFloat32(input).Write(data + (i * sizeof(Float16)));
}
for (int i = 4; i < 8; i++) {
WriteUnalignedValue<Float16>(data + (i * sizeof(Float16)),
Float16::FromFloat32(0));
}
#endif
}
void f16x8_demote_f64x2_zero_wrapper(Address data) {
#if V8_TARGET_BIG_ENDIAN
for (int i = 1, j = 7; i >= 0; i--, j--) {
double input = ReadUnalignedValue<double>(data + (i * sizeof(double)));
WriteUnalignedValue<uint16_t>(data + (j * sizeof(uint16_t)),
DoubleToFloat16(input));
}
for (int i = 0; i < 6; i++) {
WriteUnalignedValue<Float16>(data + (i * sizeof(Float16)),
Float16::FromFloat32(0));
}
#else
for (int i = 0; i < 2; i++) {
double input = ReadUnalignedValue<double>(data + (i * sizeof(double)));
WriteUnalignedValue<uint16_t>(data + (i * sizeof(uint16_t)),
DoubleToFloat16(input));
}
for (int i = 2; i < 8; i++) {
WriteUnalignedValue<Float16>(data + (i * sizeof(Float16)),
Float16::FromFloat32(0));
}
#endif
}
template <float (*float_fma_op)(float, float, float)>
void simd_float16_fma_wrapper(Address data) {
constexpr int n = kSimd128Size / sizeof(Float16);
for (int i = 0; i < n; i++) {
Address offset = data + i * sizeof(Float16);
Float16 a = Float16::Read(offset);
Float16 b = Float16::Read(offset + kSimd128Size);
Float16 c = Float16::Read(offset + 2 * kSimd128Size);
float value = float_fma_op(a.ToFloat32(), b.ToFloat32(), c.ToFloat32());
Float16::FromFloat32(value).Write(offset);
}
}
float Qfma(float a, float b, float c) { return a * b + c; }
void f16x8_qfma_wrapper(Address data) {
return simd_float16_fma_wrapper<&Qfma>(data);
}
float Qfms(float a, float b, float c) { return -(a * b) + c; }
void f16x8_qfms_wrapper(Address data) {
return simd_float16_fma_wrapper<&Qfms>(data);
}
namespace {
inline uint8_t* EffectiveAddress(Tagged<WasmTrustedInstanceData> trusted_data,
uint32_t mem_index, uintptr_t index) {
return trusted_data->memory_base(mem_index) + index;
}
template <typename V>
V ReadAndIncrementOffset(Address data, size_t* offset) {
V result = ReadUnalignedValue<V>(data + *offset);
*offset += sizeof(V);
return result;
}
constexpr int32_t kSuccess = 1;
constexpr int32_t kOutOfBounds = 0;
}
int32_t memory_init_wrapper(Address trusted_data_addr, uint32_t mem_index,
uintptr_t dst, uint32_t src, uint32_t seg_index,
uint32_t size) {
DisallowGarbageCollection no_gc;
Tagged<WasmTrustedInstanceData> trusted_data =
TrustedCast<WasmTrustedInstanceData>(Tagged<Object>{trusted_data_addr});
uint64_t mem_size = trusted_data->memory_size(mem_index);
if (!base::IsInBounds<uint64_t>(dst, size, mem_size)) return kOutOfBounds;
uint32_t seg_size = trusted_data->data_segment_sizes()->get(seg_index);
if (!base::IsInBounds<uint32_t>(src, size, seg_size)) return kOutOfBounds;
uint8_t* seg_start = reinterpret_cast<uint8_t*>(
trusted_data->data_segment_starts()->get(seg_index));
std::memcpy(EffectiveAddress(trusted_data, mem_index, dst), seg_start + src,
size);
return kSuccess;
}
int32_t memory_copy_wrapper(Address trusted_data_addr, uint32_t dst_mem_index,
uint32_t src_mem_index, uintptr_t dst,
uintptr_t src, uintptr_t size) {
DisallowGarbageCollection no_gc;
Tagged<WasmTrustedInstanceData> trusted_data =
TrustedCast<WasmTrustedInstanceData>(Tagged<Object>{trusted_data_addr});
size_t dst_mem_size = trusted_data->memory_size(dst_mem_index);
size_t src_mem_size = trusted_data->memory_size(src_mem_index);
static_assert(std::is_same_v<size_t, uintptr_t>);
if (!base::IsInBounds<size_t>(dst, size, dst_mem_size)) return kOutOfBounds;
if (!base::IsInBounds<size_t>(src, size, src_mem_size)) return kOutOfBounds;
std::memmove(EffectiveAddress(trusted_data, dst_mem_index, dst),
EffectiveAddress(trusted_data, src_mem_index, src), size);
return kSuccess;
}
int32_t memory_fill_wrapper(Address trusted_data_addr, uint32_t mem_index,
uintptr_t dst, uint8_t value, uintptr_t size) {
DisallowGarbageCollection no_gc;
Tagged<WasmTrustedInstanceData> trusted_data =
TrustedCast<WasmTrustedInstanceData>(Tagged<Object>{trusted_data_addr});
uint64_t mem_size = trusted_data->memory_size(mem_index);
if (!base::IsInBounds<uint64_t>(dst, size, mem_size)) return kOutOfBounds;
std::memset(EffectiveAddress(trusted_data, mem_index, dst), value, size);
return kSuccess;
}
namespace {
inline void* ArrayElementAddress(Address array, uint32_t index,
int element_size_bytes) {
return reinterpret_cast<void*>(array + WasmArray::kHeaderSize -
kHeapObjectTag + index * element_size_bytes);
}
inline void* ArrayElementAddress(Tagged<WasmArray> array, uint32_t index,
int element_size_bytes) {
return ArrayElementAddress(array.ptr(), index, element_size_bytes);
}
}
void array_copy_wrapper(Address raw_dst_array, uint32_t dst_index,
Address raw_src_array, uint32_t src_index,
uint32_t length) {
DCHECK_GT(length, 0);
DisallowGarbageCollection no_gc;
Tagged<WasmArray> dst_array = Cast<WasmArray>(Tagged<Object>(raw_dst_array));
Tagged<WasmArray> src_array = Cast<WasmArray>(Tagged<Object>(raw_src_array));
bool overlapping_ranges =
dst_array.ptr() == src_array.ptr() &&
(dst_index < src_index ? dst_index + length > src_index
: src_index + length > dst_index);
wasm::CanonicalValueType element_type =
src_array->map()->wasm_type_info()->element_type();
if (element_type.is_reference()) {
ObjectSlot dst_slot = dst_array->ElementSlot(dst_index);
ObjectSlot src_slot = src_array->ElementSlot(src_index);
Heap* heap = Isolate::Current()->heap();
if (overlapping_ranges) {
heap->MoveRange(dst_array, dst_slot, src_slot, length,
UPDATE_WRITE_BARRIER);
} else {
heap->CopyRange(dst_array, dst_slot, src_slot, length,
UPDATE_WRITE_BARRIER);
}
} else {
int element_size_bytes = element_type.value_kind_size();
void* dst = ArrayElementAddress(dst_array, dst_index, element_size_bytes);
void* src = ArrayElementAddress(src_array, src_index, element_size_bytes);
size_t copy_size = length * element_size_bytes;
if (overlapping_ranges) {
MemMove(dst, src, copy_size);
} else {
MemCopy(dst, src, copy_size);
}
}
}
void array_fill_wrapper(Address raw_array, uint32_t index, uint32_t length,
uint32_t emit_write_barrier, uint32_t raw_type,
Address initial_value_addr) {
DisallowGarbageCollection no_gc;
ValueType type = ValueType::FromRawBitField(raw_type);
int8_t* initial_element_address = reinterpret_cast<int8_t*>(
ArrayElementAddress(raw_array, index, type.value_kind_size()));
const int bytes_to_set = length * type.value_kind_size();
CHECK_GE(static_cast<size_t>(bytes_to_set), sizeof(int64_t));
switch (type.kind()) {
case kI64:
case kF64: {
int64_t initial_value =
base::ReadUnalignedValue<int64_t>(initial_value_addr);
if (initial_value == 0) {
std::memset(initial_element_address, 0, bytes_to_set);
return;
}
base::WriteUnalignedValue<int64_t>(
reinterpret_cast<Address>(initial_element_address), initial_value);
break;
}
case kI32:
case kF32: {
int32_t initial_value = *reinterpret_cast<int32_t*>(initial_value_addr);
if (initial_value == 0) {
std::memset(initial_element_address, 0, bytes_to_set);
return;
}
int32_t* base = reinterpret_cast<int32_t*>(initial_element_address);
base[0] = base[1] = initial_value;
break;
}
case kF16:
case kI16: {
int16_t initial_value = *reinterpret_cast<int32_t*>(initial_value_addr);
if (initial_value == 0) {
std::memset(initial_element_address, 0, bytes_to_set);
return;
}
int16_t* base = reinterpret_cast<int16_t*>(initial_element_address);
base[0] = base[1] = base[2] = base[3] = initial_value;
break;
}
case kI8: {
int8_t initial_value = *reinterpret_cast<int32_t*>(initial_value_addr);
if (initial_value == 0) {
std::memset(initial_element_address, 0, bytes_to_set);
return;
}
int8_t* base = reinterpret_cast<int8_t*>(initial_element_address);
for (size_t i = 0; i < sizeof(int64_t); i++) {
base[i] = initial_value;
}
break;
}
case kRefNull:
case kRef: {
intptr_t uncompressed_pointer =
base::ReadUnalignedValue<intptr_t>(initial_value_addr);
if constexpr (kTaggedSize == 4) {
int32_t* base = reinterpret_cast<int32_t*>(initial_element_address);
base[0] = base[1] = static_cast<int32_t>(uncompressed_pointer);
} else {
base::WriteUnalignedValue(
reinterpret_cast<Address>(initial_element_address),
uncompressed_pointer);
}
break;
}
case kS128:
DCHECK_EQ(base::ReadUnalignedValue<int64_t>(initial_value_addr), 0);
std::memset(initial_element_address, 0, bytes_to_set);
return;
case kVoid:
case kTop:
case kBottom:
UNREACHABLE();
}
int bytes_already_set = sizeof(int64_t);
while (bytes_already_set * 2 <= bytes_to_set) {
std::memcpy(initial_element_address + bytes_already_set,
initial_element_address, bytes_already_set);
bytes_already_set *= 2;
}
if (bytes_already_set < bytes_to_set) {
std::memcpy(initial_element_address + bytes_already_set,
initial_element_address, bytes_to_set - bytes_already_set);
}
if (emit_write_barrier) {
DCHECK(type.is_reference());
Tagged<WasmArray> array = Cast<WasmArray>(Tagged<Object>(raw_array));
Isolate* isolate = Isolate::Current();
ObjectSlot start(reinterpret_cast<Address>(initial_element_address));
ObjectSlot end(
reinterpret_cast<Address>(initial_element_address + bytes_to_set));
WriteBarrier::ForRange(isolate->heap(), array, start, end);
}
}
double flat_string_to_f64(Address string_address) {
Tagged<String> s = Cast<String>(Tagged<Object>(string_address));
return FlatStringToDouble(s, ALLOW_TRAILING_JUNK,
std::numeric_limits<double>::quiet_NaN());
}
void start_stack(Isolate* isolate, wasm::StackMemory* to, Address sp,
Address fp, Address pc) {
wasm::StackMemory* from = isolate->isolate_data()->active_stack();
if (v8_flags.trace_wasm_stack_switching) {
PrintF("Switch from stack %d to %d (start)\n", from->id(), to->id());
}
isolate->SwitchStacks<JumpBuffer::Inactive, JumpBuffer::Suspended>(
from, to, sp, fp, pc);
}
int32_t suspender_has_js_frames(Isolate* isolate) {
wasm::StackMemory* from_stack = isolate->isolate_data()->active_stack();
Tagged<WasmSuspenderObject> suspender =
isolate->isolate_data()->active_suspender();
Tagged<WasmSuspenderObject> parent = suspender->parent();
wasm::StackMemory* to_stack = parent->stack();
for (wasm::StackMemory* stack = from_stack; stack != to_stack;
stack = stack->jmpbuf()->parent) {
if (stack->jmpbuf()->is_on_central_stack) {
return true;
}
}
return false;
}
void suspend_stack(Isolate* isolate, wasm::StackMemory* to, Address sp,
Address fp, Address pc) {
wasm::StackMemory* from = isolate->isolate_data()->active_stack();
auto suspender = isolate->isolate_data()->active_suspender();
suspender->set_stack(isolate, from);
if (v8_flags.trace_wasm_stack_switching) {
PrintF("Switch from stack %d to %d (suspend)\n", from->id(), to->id());
}
isolate->SwitchStacks<JumpBuffer::Suspended, JumpBuffer::Inactive>(
from, to, sp, fp, pc);
}
void resume_jspi_stack(Isolate* isolate, wasm::StackMemory* to, Address sp,
Address fp, Address pc, Address suspender_raw) {
Tagged<Object> suspender_obj(suspender_raw);
auto suspender = TrustedCast<WasmSuspenderObject>(suspender_obj);
Tagged<WasmSuspenderObject> active_suspender =
isolate->isolate_data()->active_suspender();
suspender->set_parent(active_suspender);
wasm::StackMemory* from = isolate->isolate_data()->active_stack();
if (v8_flags.trace_wasm_stack_switching) {
PrintF("Switch from stack %d to %d (resume)\n", from->id(), to->id());
}
isolate->isolate_data()->set_active_suspender(suspender);
isolate->SwitchStacks<JumpBuffer::Inactive, JumpBuffer::Suspended>(
from, to, sp, fp, pc);
}
void resume_wasmfx_stack(Isolate* isolate, wasm::StackMemory* to, Address sp,
Address fp, Address pc) {
wasm::StackMemory* from = isolate->isolate_data()->active_stack();
to->set_current_continuation({});
if (v8_flags.trace_wasm_stack_switching) {
PrintF("Switch from stack %d to %d (resume)\n", from->id(), to->id());
}
isolate->SwitchStacks<JumpBuffer::Inactive, JumpBuffer::Suspended>(
from, to, sp, fp, pc);
}
Address suspend_wasmfx_stack(Isolate* isolate, Address sp, Address fp,
Address pc, Address wanted_tag_raw,
Address cont_raw) {
Tagged<Object> tag_obj(wanted_tag_raw);
auto wanted_tag = TrustedCast<WasmExceptionTag>(tag_obj);
Tagged<Object> cont_obj(cont_raw);
auto cont = TrustedCast<WasmContinuationObject>(cont_obj);
wasm::StackMemory* from = isolate->isolate_data()->active_stack();
cont->set_stack(isolate, from);
from->set_current_continuation(cont);
wasm::StackMemory* to = from->jmpbuf()->parent;
bool found = false;
while (true) {
StackFrameIterator it(isolate, to);
CHECK_EQ(it.frame()->type(), StackFrame::WASM_STACK_EXIT);
it.Advance();
CHECK(it.frame()->is_wasm());
WasmCode* wasm_code =
wasm::GetWasmCodeManager()->LookupCode(isolate, it.frame()->pc());
base::Vector<const WasmCode::EffectHandler> effect_handlers =
wasm_code->effect_handlers();
Tagged<Object> trusted_instance_data_obj(base::Memory<Address>(
it.frame()->fp() + WasmFrameConstants::kWasmInstanceDataOffset));
auto trusted_instance_data =
TrustedCast<WasmTrustedInstanceData>(trusted_instance_data_obj);
for (const auto& handler : effect_handlers) {
auto tag = trusted_instance_data->tags_table()->get(handler.tag_index);
if (wasm_code->instruction_start() + handler.call_offset ==
it.frame()->pc() &&
tag == wanted_tag) {
found = true;
to->jmpbuf()->pc =
wasm_code->instruction_start() + handler.handler_offset;
to->jmpbuf()->sp = it.frame()->sp();
to->jmpbuf()->fp = it.frame()->fp();
break;
}
}
if (found) break;
if (to->jmpbuf()->is_on_central_stack) {
return kNullAddress;
}
to = to->jmpbuf()->parent;
}
if (!found) {
return kNullAddress;
}
if (v8_flags.trace_wasm_stack_switching) {
PrintF("Switch from stack %d to %d (suspend)\n", from->id(), to->id());
}
isolate->SwitchStacks<JumpBuffer::Suspended, JumpBuffer::Inactive>(
from, to, sp, fp, pc);
return reinterpret_cast<Address>(to);
}
void return_stack(Isolate* isolate, wasm::StackMemory* to) {
wasm::StackMemory* from = isolate->isolate_data()->active_stack();
if (v8_flags.trace_wasm_stack_switching) {
PrintF("Switch from stack %d to %d (return)\n", from->id(), to->id());
}
isolate->SwitchStacks<JumpBuffer::Retired, JumpBuffer::Inactive>(
from, to, kNullAddress, kNullAddress, kNullAddress);
isolate->RetireWasmStack(from);
}
void return_jspi_stack(Isolate* isolate, wasm::StackMemory* to) {
Tagged<WasmSuspenderObject> suspender =
isolate->isolate_data()->active_suspender();
suspender->set_stack(isolate, nullptr);
return_stack(isolate, to);
}
void return_wasmfx_stack(Isolate* isolate, wasm::StackMemory* to) {
return_stack(isolate, to);
}
intptr_t switch_to_the_central_stack(Isolate* isolate, uintptr_t current_sp) {
ThreadLocalTop* thread_local_top = isolate->thread_local_top();
StackGuard* stack_guard = isolate->stack_guard();
auto secondary_stack_limit = stack_guard->real_jslimit();
stack_guard->SetStackLimitForStackSwitching(
thread_local_top->central_stack_limit_);
thread_local_top->secondary_stack_limit_ = secondary_stack_limit;
thread_local_top->secondary_stack_sp_ = current_sp;
thread_local_top->is_on_central_stack_flag_ = true;
auto counter = isolate->wasm_switch_to_the_central_stack_counter();
isolate->set_wasm_switch_to_the_central_stack_counter(counter + 1);
DCHECK_NE(thread_local_top->central_stack_sp_, kNullAddress);
return thread_local_top->central_stack_sp_;
}
void switch_from_the_central_stack(Isolate* isolate) {
ThreadLocalTop* thread_local_top = isolate->thread_local_top();
CHECK_NE(thread_local_top->secondary_stack_sp_, 0);
CHECK_NE(thread_local_top->secondary_stack_limit_, 0);
auto secondary_stack_limit = thread_local_top->secondary_stack_limit_;
thread_local_top->secondary_stack_limit_ = 0;
thread_local_top->secondary_stack_sp_ = 0;
thread_local_top->is_on_central_stack_flag_ = false;
StackGuard* stack_guard = isolate->stack_guard();
stack_guard->SetStackLimitForStackSwitching(secondary_stack_limit);
}
intptr_t switch_to_the_central_stack_for_js(Isolate* isolate, Address fp) {
ThreadLocalTop* thread_local_top = isolate->thread_local_top();
StackGuard* stack_guard = isolate->stack_guard();
wasm::StackMemory* stack = isolate->isolate_data()->active_stack();
Address central_stack_sp = thread_local_top->central_stack_sp_;
DCHECK_NE(central_stack_sp, kNullAddress);
stack->set_stack_switch_info(fp, central_stack_sp);
stack_guard->SetStackLimitForStackSwitching(
thread_local_top->central_stack_limit_);
thread_local_top->is_on_central_stack_flag_ = true;
return central_stack_sp;
}
void switch_from_the_central_stack_for_js(Isolate* isolate) {
wasm::StackMemory* stack = isolate->isolate_data()->active_stack();
stack->clear_stack_switch_info();
ThreadLocalTop* thread_local_top = isolate->thread_local_top();
thread_local_top->is_on_central_stack_flag_ = false;
StackGuard* stack_guard = isolate->stack_guard();
stack_guard->SetStackLimitForStackSwitching(
reinterpret_cast<uintptr_t>(stack->jslimit()));
}
Address grow_stack(Isolate* isolate, void* current_sp, size_t frame_size,
size_t gap, Address current_fp) {
StackLimitCheck check(isolate);
if (check.WasmHasOverflowed(gap)) {
wasm::StackMemory* active_stack = isolate->isolate_data()->active_stack();
if (isolate->IsOnCentralStack()) {
return 0;
}
DCHECK(active_stack->IsActive());
size_t min =
gap + frame_size + StackMemory::JSGrowableStackLimitMarginKB() * KB;
if (!active_stack->Grow(current_fp, min)) {
return 0;
}
Address new_sp = active_stack->base() - frame_size;
std::memcpy(reinterpret_cast<void*>(new_sp), current_sp, frame_size);
#if V8_TARGET_ARCH_ARM64
Address new_fp =
new_sp + (current_fp - reinterpret_cast<Address>(current_sp));
Address old_pc_address = current_fp + CommonFrameConstants::kCallerPCOffset;
Address new_pc_address = new_fp + CommonFrameConstants::kCallerPCOffset;
Address old_signed_pc = base::Memory<Address>(old_pc_address);
Address new_signed_pc = PointerAuthentication::MoveSignedPC(
isolate, old_signed_pc, new_pc_address + kSystemPointerSize,
old_pc_address + kSystemPointerSize);
WriteUnalignedValue<Address>(new_pc_address, new_signed_pc);
#endif
isolate->stack_guard()->SetStackLimitForStackSwitching(
reinterpret_cast<uintptr_t>(active_stack->jslimit()));
return new_sp;
}
return 0;
}
Address shrink_stack(Isolate* isolate) {
wasm::StackMemory* active_stack = isolate->isolate_data()->active_stack();
if (active_stack->jmpbuf()->parent == nullptr) {
return 0;
}
DCHECK(active_stack->IsActive());
Address old_fp = active_stack->Shrink();
isolate->stack_guard()->SetStackLimitForStackSwitching(
reinterpret_cast<uintptr_t>(active_stack->jslimit()));
return old_fp;
}
Address load_old_fp(Isolate* isolate) {
wasm::StackMemory* active_stack = isolate->isolate_data()->active_stack();
if (active_stack->jmpbuf()->parent == nullptr) {
return 0;
}
DCHECK_EQ(active_stack->jmpbuf()->state, wasm::JumpBuffer::Active);
return active_stack->old_fp();
}
}
#undef V8_WITH_SANITIZER
#undef RESET_THREAD_IN_WASM_FLAG_FOR_ASAN_ON_WINDOWS