#ifndef V8_COMMON_CODE_MEMORY_ACCESS_H_
#define V8_COMMON_CODE_MEMORY_ACCESS_H_
#include <map>
#include <optional>
#include "include/v8-internal.h"
#include "include/v8-platform.h"
#include "src/base/build_config.h"
#include "src/base/macros.h"
#include "src/base/memory.h"
#include "src/base/platform/mutex.h"
#include "src/common/globals.h"
namespace v8 {
namespace internal {
class RwxMemoryWriteScopeForTesting;
namespace wasm {
class CodeSpaceWriteScope;
}
#if V8_HAS_PKU_JIT_WRITE_PROTECT
#define THREAD_ISOLATION_ALIGN_SZ 0x1000
#define THREAD_ISOLATION_ALIGN alignas(THREAD_ISOLATION_ALIGN_SZ)
#define THREAD_ISOLATION_ALIGN_OFFSET_MASK (THREAD_ISOLATION_ALIGN_SZ - 1)
#define THREAD_ISOLATION_FILL_PAGE_SZ(size) \
((THREAD_ISOLATION_ALIGN_SZ - \
((size) & THREAD_ISOLATION_ALIGN_OFFSET_MASK)) % \
THREAD_ISOLATION_ALIGN_SZ)
#else
#define THREAD_ISOLATION_ALIGN_SZ 0
#define THREAD_ISOLATION_ALIGN
#define THREAD_ISOLATION_FILL_PAGE_SZ(size) 0
#endif
class V8_NODISCARD RwxMemoryWriteScope {
public:
V8_INLINE explicit RwxMemoryWriteScope(const char* comment);
V8_INLINE ~RwxMemoryWriteScope();
RwxMemoryWriteScope(const RwxMemoryWriteScope&) = delete;
RwxMemoryWriteScope& operator=(const RwxMemoryWriteScope&) = delete;
V8_INLINE static bool IsSupported();
#if V8_HAS_PKU_JIT_WRITE_PROTECT
static int memory_protection_key();
static bool IsPKUWritable();
#endif
private:
friend class RwxMemoryWriteScopeForTesting;
friend class wasm::CodeSpaceWriteScope;
friend class WritableJumpTablePair;
V8_INLINE static void SetWritable();
V8_INLINE static void SetExecutable();
};
class WritableJitPage;
class WritableJitAllocation;
class WritableJumpTablePair;
class V8_EXPORT ThreadIsolation {
public:
static bool Enabled();
static void Initialize(ThreadIsolatedAllocator* allocator);
enum class JitAllocationType {
kInstructionStream,
kWasmCode,
kWasmJumpTable,
kWasmFarJumpTable,
kWasmLazyCompileTable,
};
static void RegisterJitPage(Address address, size_t size);
static void UnregisterJitPage(Address address, size_t size);
V8_NODISCARD static bool MakeExecutable(Address address, size_t size);
static WritableJitAllocation RegisterJitAllocation(
Address addr, size_t size, JitAllocationType type,
bool enforce_write_api = false);
static WritableJitAllocation RegisterInstructionStreamAllocation(
Address addr, size_t size, bool enforce_write_api = false);
static void RegisterJitAllocations(Address start,
const std::vector<size_t>& sizes,
JitAllocationType type);
static WritableJitAllocation LookupJitAllocation(
Address addr, size_t size, JitAllocationType type,
bool enforce_write_api = false);
#ifdef V8_ENABLE_WEBASSEMBLY
static WritableJumpTablePair LookupJumpTableAllocations(
Address jump_table_address, size_t jump_table_size,
Address far_jump_table_address, size_t far_jump_table_size);
#endif
static WritableJitPage LookupWritableJitPage(Address addr, size_t size);
static void UnregisterWasmAllocation(Address addr, size_t size);
static bool CanLookupStartOfJitAllocationAt(Address inner_pointer);
static std::optional<Address> StartOfJitAllocationAt(Address inner_pointer);
V8_NODISCARD static bool WriteProtectMemory(
Address addr, size_t size, PageAllocator::Permission page_permissions);
static void RegisterJitAllocationForTesting(Address obj, size_t size);
static void UnregisterJitAllocationForTesting(Address addr, size_t size);
#if V8_HAS_PKU_JIT_WRITE_PROTECT
static int pkey() { return trusted_data_.pkey; }
static bool PkeyIsAvailable() { return trusted_data_.pkey != -1; }
#endif
#if DEBUG
static bool initialized() { return trusted_data_.initialized; }
static void CheckTrackedMemoryEmpty();
#endif
template <class T>
struct StlAllocator {
typedef T value_type;
StlAllocator() = default;
template <class U>
explicit StlAllocator(const StlAllocator<U>&) noexcept {}
value_type* allocate(size_t n) {
if (Enabled()) {
return static_cast<value_type*>(
ThreadIsolation::allocator()->Allocate(n * sizeof(value_type)));
} else {
return static_cast<value_type*>(::operator new(n * sizeof(T)));
}
}
void deallocate(value_type* ptr, size_t n) {
if (Enabled()) {
ThreadIsolation::allocator()->Free(ptr);
} else {
::operator delete(ptr);
}
}
};
class JitAllocation {
public:
explicit JitAllocation(size_t size, JitAllocationType type)
: size_(size), type_(type) {}
size_t Size() const { return size_; }
JitAllocationType Type() const { return type_; }
private:
size_t size_;
JitAllocationType type_;
};
class JitPage;
class V8_EXPORT JitPageReference {
public:
JitPageReference(class JitPage* page, Address address);
JitPageReference(JitPageReference&&) V8_NOEXCEPT = default;
JitPageReference(const JitPageReference&) = delete;
JitPageReference& operator=(const JitPageReference&) = delete;
base::Address Address() const { return address_; }
size_t Size() const;
base::Address End() const { return Address() + Size(); }
JitAllocation& RegisterAllocation(base::Address addr, size_t size,
JitAllocationType type);
JitAllocation& LookupAllocation(base::Address addr, size_t size,
JitAllocationType type);
bool Contains(base::Address addr, size_t size,
JitAllocationType type) const;
void UnregisterAllocation(base::Address addr);
void UnregisterAllocationsExcept(base::Address start, size_t size,
const std::vector<base::Address>& addr);
void UnregisterRange(base::Address addr, size_t size);
base::Address StartOfAllocationAt(base::Address inner_pointer);
std::pair<base::Address, JitAllocation&> AllocationContaining(
base::Address addr);
bool Empty() const { return jit_page_->allocations_.empty(); }
void Shrink(class JitPage* tail);
void Expand(size_t offset);
void Merge(JitPageReference& next);
class JitPage* JitPage() { return jit_page_; }
private:
base::MutexGuard page_lock_;
class JitPage* jit_page_;
base::Address address_;
};
class JitPage {
public:
explicit JitPage(size_t size) : size_(size) {}
~JitPage();
private:
base::Mutex mutex_;
typedef std::map<Address, JitAllocation, std::less<Address>,
StlAllocator<std::pair<const Address, JitAllocation>>>
AllocationMap;
AllocationMap allocations_;
size_t size_;
friend class JitPageReference;
friend bool ThreadIsolation::CanLookupStartOfJitAllocationAt(Address);
};
private:
static ThreadIsolatedAllocator* allocator() {
return trusted_data_.allocator;
}
typedef std::map<Address, JitPage*, std::less<Address>,
StlAllocator<std::pair<const Address, JitPage*>>>
JitPageMap;
struct THREAD_ISOLATION_ALIGN TrustedData {
ThreadIsolatedAllocator* allocator = nullptr;
#if V8_HAS_PKU_JIT_WRITE_PROTECT
int pkey = -1;
#endif
base::Mutex* jit_pages_mutex_;
JitPageMap* jit_pages_;
#if DEBUG
bool initialized = false;
#endif
};
static struct TrustedData trusted_data_;
static_assert(THREAD_ISOLATION_ALIGN_SZ == 0 ||
sizeof(trusted_data_) == THREAD_ISOLATION_ALIGN_SZ);
template <typename T, typename... Args>
static void ConstructNew(T** ptr, Args&&... args);
template <typename T>
static void Delete(T* ptr);
static JitPageReference LookupJitPage(Address addr, size_t size);
static JitPageReference LookupJitPageLocked(Address addr, size_t size);
static std::optional<JitPageReference> TryLookupJitPage(Address addr,
size_t size);
static std::optional<JitPageReference> TryLookupJitPageLocked(Address addr,
size_t size);
static JitPageReference SplitJitPageLocked(Address addr, size_t size);
static JitPageReference SplitJitPage(Address addr, size_t size);
static std::pair<JitPageReference, JitPageReference> SplitJitPages(
Address addr1, size_t size1, Address addr2, size_t size2);
template <class T>
friend struct StlAllocator;
friend class WritableJitPage;
friend class WritableJitAllocation;
friend class WritableJumpTablePair;
};
class WritableJitAllocation {
public:
WritableJitAllocation(const WritableJitAllocation&) = delete;
WritableJitAllocation& operator=(const WritableJitAllocation&) = delete;
V8_INLINE ~WritableJitAllocation();
static WritableJitAllocation ForInstructionStream(
Tagged<InstructionStream> istream);
static V8_INLINE WritableJitAllocation ForNonExecutableMemory(
Address addr, size_t size, ThreadIsolation::JitAllocationType type);
#ifdef V8_ENABLE_SPARKPLUG_PLUS
static V8_INLINE WritableJitAllocation ForPatchableBaselineJIT(Address addr,
size_t size);
#endif
template <typename T, size_t offset>
V8_INLINE void WriteHeaderSlot(T value);
template <typename T, size_t offset>
V8_INLINE void WriteHeaderSlot(Tagged<T> value, ReleaseStoreTag);
template <typename T, size_t offset>
V8_INLINE void WriteHeaderSlot(Tagged<T> value, RelaxedStoreTag);
template <typename T, size_t offset>
V8_INLINE void WriteProtectedPointerHeaderSlot(Tagged<T> value,
ReleaseStoreTag);
template <typename T, size_t offset>
V8_INLINE void WriteProtectedPointerHeaderSlot(Tagged<T> value,
RelaxedStoreTag);
template <typename T>
V8_INLINE void WriteHeaderSlot(Address address, T value, RelaxedStoreTag);
V8_INLINE void CopyCode(size_t dst_offset, const uint8_t* src,
size_t num_bytes);
V8_INLINE void CopyData(size_t dst_offset, const uint8_t* src,
size_t num_bytes);
template <typename T>
V8_INLINE void WriteUnalignedValue(Address address, T value);
template <typename T>
V8_INLINE void WriteValue(Address address, T value);
template <typename T>
V8_INLINE void WriteValue(Address address, T value, RelaxedStoreTag);
V8_INLINE void ClearBytes(size_t offset, size_t len);
#ifdef V8_ENABLE_WEBASSEMBLY
void UpdateWasmCodePointer(WasmCodePointer code_pointer,
uint64_t signature_hash);
#endif
Address address() const { return address_; }
size_t size() const { return allocation_.Size(); }
private:
enum class JitAllocationSource {
kRegister,
kLookup,
};
V8_INLINE WritableJitAllocation(Address addr, size_t size,
ThreadIsolation::JitAllocationType type,
JitAllocationSource source,
bool enforce_write_api = false);
V8_INLINE WritableJitAllocation(Address addr, size_t size,
ThreadIsolation::JitAllocationType type,
bool enforce_write_api);
#ifdef V8_ENABLE_SPARKPLUG_PLUS
V8_INLINE WritableJitAllocation(Address addr, size_t size);
#endif
ThreadIsolation::JitPageReference& page_ref() { return page_ref_.value(); }
V8_INLINE std::optional<RwxMemoryWriteScope> WriteScopeForApiEnforcement()
const;
const Address address_;
std::optional<RwxMemoryWriteScope> write_scope_;
std::optional<ThreadIsolation::JitPageReference> page_ref_;
const ThreadIsolation::JitAllocation allocation_;
bool enforce_write_api_ = false;
friend class ThreadIsolation;
friend class WritableJitPage;
friend class WritableJumpTablePair;
};
class WritableFreeSpace {
public:
static V8_INLINE WritableFreeSpace ForNonExecutableMemory(base::Address addr,
size_t size);
WritableFreeSpace(const WritableFreeSpace&) = delete;
WritableFreeSpace& operator=(const WritableFreeSpace&) = delete;
V8_INLINE ~WritableFreeSpace();
template <typename T, size_t offset>
V8_INLINE void WriteHeaderSlot(Tagged<T> value, RelaxedStoreTag) const;
template <size_t offset>
void ClearTagged(size_t count) const;
base::Address Address() const { return address_; }
int Size() const { return size_; }
bool Executable() const { return executable_; }
private:
WritableFreeSpace(base::Address addr, size_t size, bool executable);
const base::Address address_;
const int size_;
const bool executable_;
friend class WritableJitPage;
};
extern template void WritableFreeSpace::ClearTagged<kTaggedSize>(
size_t count) const;
extern template void WritableFreeSpace::ClearTagged<2 * kTaggedSize>(
size_t count) const;
class WritableJitPage {
public:
V8_INLINE WritableJitPage(Address addr, size_t size);
WritableJitPage(const WritableJitPage&) = delete;
WritableJitPage& operator=(const WritableJitPage&) = delete;
V8_INLINE ~WritableJitPage();
friend class ThreadIsolation;
V8_INLINE WritableJitAllocation LookupAllocationContaining(Address addr);
V8_INLINE WritableFreeSpace FreeRange(Address addr, size_t size);
bool Empty() const { return page_ref_.Empty(); }
private:
RwxMemoryWriteScope write_scope_;
ThreadIsolation::JitPageReference page_ref_;
};
#ifdef V8_ENABLE_WEBASSEMBLY
class V8_EXPORT_PRIVATE WritableJumpTablePair {
public:
WritableJitAllocation& jump_table() { return writable_jump_table_; }
WritableJitAllocation& far_jump_table() { return writable_far_jump_table_; }
~WritableJumpTablePair();
WritableJumpTablePair(const WritableJumpTablePair&) = delete;
WritableJumpTablePair& operator=(const WritableJumpTablePair&) = delete;
static WritableJumpTablePair ForTesting(Address jump_table_address,
size_t jump_table_size,
Address far_jump_table_address,
size_t far_jump_table_size);
private:
V8_INLINE WritableJumpTablePair(Address jump_table_address,
size_t jump_table_size,
Address far_jump_table_address,
size_t far_jump_table_size);
struct ForTestingTag {};
WritableJumpTablePair(Address jump_table_address, size_t jump_table_size,
Address far_jump_table_address,
size_t far_jump_table_size, ForTestingTag);
WritableJitAllocation writable_jump_table_;
WritableJitAllocation writable_far_jump_table_;
RwxMemoryWriteScope write_scope_;
std::optional<std::pair<ThreadIsolation::JitPageReference,
ThreadIsolation::JitPageReference>>
jump_table_pages_;
friend class ThreadIsolation;
};
#endif
template <class T>
bool operator==(const ThreadIsolation::StlAllocator<T>&,
const ThreadIsolation::StlAllocator<T>&) {
return true;
}
template <class T>
bool operator!=(const ThreadIsolation::StlAllocator<T>&,
const ThreadIsolation::StlAllocator<T>&) {
return false;
}
class V8_NODISCARD V8_ALLOW_UNUSED NopRwxMemoryWriteScope final {
public:
V8_INLINE NopRwxMemoryWriteScope() = default;
V8_INLINE explicit NopRwxMemoryWriteScope(const char* comment) {
}
};
class V8_NODISCARD RwxMemoryWriteScopeForTesting final
: public RwxMemoryWriteScope {
public:
V8_EXPORT_PRIVATE RwxMemoryWriteScopeForTesting();
V8_EXPORT_PRIVATE ~RwxMemoryWriteScopeForTesting();
RwxMemoryWriteScopeForTesting(const RwxMemoryWriteScopeForTesting&) = delete;
RwxMemoryWriteScopeForTesting& operator=(
const RwxMemoryWriteScopeForTesting&) = delete;
};
#if V8_HEAP_USE_PTHREAD_JIT_WRITE_PROTECT
using CFIMetadataWriteScope = NopRwxMemoryWriteScope;
#else
using CFIMetadataWriteScope = RwxMemoryWriteScope;
#endif
#ifdef V8_ENABLE_MEMORY_SEALING
using DiscardSealedMemoryScope = RwxMemoryWriteScope;
#else
using DiscardSealedMemoryScope = NopRwxMemoryWriteScope;
#endif
}
}
#endif