* Copyright (c) 2010-2023 Google, Inc. All rights reserved.
* Copyright (c) 2001-2010 VMware, Inc. All rights reserved.
* **********************************************************/
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* * Neither the name of VMware, Inc. nor the names of its contributors may be
* used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
* DAMAGE.
*/
* heap.c - heap manager
*/
#include "globals.h"
#include <limits.h>
#include "fragment.h"
#include "link.h"
#include "instr.h"
#include "fcache.h"
#ifdef UNIX
# include "memquery.h"
#endif
#ifdef DEBUG
# include "hotpatch.h"
#endif
#include "instrument.h"
#ifdef HEAP_ACCOUNTING
# ifndef DEBUG
# error HEAP_ACCOUNTING requires DEBUG
# endif
#endif
#ifdef DEBUG_MEMORY
# define CHKLVL_MEMFILL CHKLVL_DEFAULT
#endif
extern bool vm_areas_exited;
* we cannot use malloc in the middle of interpreting the client program
* because we could be in the middle of interpreting malloc, which is not
* always reentrant
*
* We have a virtual memory manager which makes sure memory is
* reserved within the application address space so that we don't have
* to fight with the application. We call os_heap_reserve to allocate
* virtual space in a single consecutive region. We later use
* os_heap_commit to get committed memory in large chunks and manage
* the chunks using a simple scheme of free lists of different sizes.
* The virtual memory manager has to store out of band information about
* used and free blocks, since of course there is no real memory to use.
* The chunks (heap units) store in band extra information both for
* used and free. However, in the allocated blocks within a unit we
* don't need to store any information since heap_free passes in the
* size; we store the next pointers for the free lists at the start of
* the free blocks themselves. We have one large reservation for most of
* our allocations, and yet another for allocations that we do not
* plan on ever freeing up on detach - the only unavoidable tombstones
* are those for thread private code system calls that may be stuck on
* callbacks. In case we run out of reserved memory we do fall back
* on requests from the OS, but any of these may fail if we are
* competing with the application.
*
* looking at dynamo behavior as of Jan 2001, most heap_alloc requests are
* for < 128 bytes, very few for larger, so we have a bunch of fixed-size
* blocks of small sizes
*
* the UINT_MAX size is a variable-length block, we keep one byte to store
* the size (again storing the next pointer when free at the start of
* what we pass to the user)
*/
static const uint BLOCK_SIZES[] = {
8,
#ifndef X64
sizeof(future_fragment_t),
#endif
24,
ALIGN_FORWARD(sizeof(fragment_t) + sizeof(indirect_linkstub_t), HEAP_ALIGNMENT),
#if defined(X64)
# ifdef DEBUG
sizeof(fragment_t) + sizeof(direct_linkstub_t) +
sizeof(cbr_fallthrough_linkstub_t),
# else
sizeof(instr_t),
# endif
#else
sizeof(fragment_t) + sizeof(direct_linkstub_t) +
sizeof(cbr_fallthrough_linkstub_t),
# ifndef DEBUG
sizeof(instr_t),
# endif
#endif
* hit this.
*/
ALIGN_FORWARD(sizeof(fragment_t) + 2 * sizeof(direct_linkstub_t),
HEAP_ALIGNMENT),
ALIGN_FORWARD(sizeof(trace_t) + 2 * sizeof(direct_linkstub_t) + sizeof(uint),
HEAP_ALIGNMENT),
ALIGN_FORWARD(sizeof(trace_t) + 3 * sizeof(direct_linkstub_t) + sizeof(uint),
HEAP_ALIGNMENT),
ALIGN_FORWARD(sizeof(trace_t) + 5 * sizeof(direct_linkstub_t) + sizeof(uint),
HEAP_ALIGNMENT),
256, 512, UINT_MAX
};
#define BLOCK_TYPES (sizeof(BLOCK_SIZES) / sizeof(uint))
#ifdef DEBUG
* gracefully handle underflow to negative values
*/
DECLARE_NEVERPROT_VAR(static int block_total_count[BLOCK_TYPES], { 0 });
DECLARE_NEVERPROT_VAR(static int block_count[BLOCK_TYPES], { 0 });
DECLARE_NEVERPROT_VAR(static int block_peak_count[BLOCK_TYPES], { 0 });
DECLARE_NEVERPROT_VAR(static int block_wasted[BLOCK_TYPES], { 0 });
DECLARE_NEVERPROT_VAR(static int block_peak_wasted[BLOCK_TYPES], { 0 });
DECLARE_NEVERPROT_VAR(static int block_align_pad[BLOCK_TYPES], { 0 });
DECLARE_NEVERPROT_VAR(static int block_peak_align_pad[BLOCK_TYPES], { 0 });
DECLARE_NEVERPROT_VAR(static bool out_of_vmheap_once, false);
#endif
#define HEADER_SIZE (sizeof(size_t))
#define VARIABLE_SIZE(p) (*(size_t *)((p)-HEADER_SIZE))
#define MEMSET_HEADER(p, value) VARIABLE_SIZE(p) = HEAP_TO_PTR_UINT(value)
#define GET_VARIABLE_ALLOCATION_SIZE(p) (VARIABLE_SIZE(p) + HEADER_SIZE)
* We start out with a small unit. Then each additional unit we
* need doubles in size, up to a maximum.
* We keep the initial units small for thread-private heaps, since with
* thousands of threads the space can add up.
*/
#define HEAP_UNIT_MIN_SIZE DYNAMO_OPTION(initial_heap_unit_size)
#define HEAP_UNIT_MAX_SIZE INTERNAL_OPTION(max_heap_unit_size)
#define GLOBAL_UNIT_MIN_SIZE DYNAMO_OPTION(initial_global_heap_unit_size)
#define GUARD_PAGE_ADJUSTMENT (dynamo_options.guard_pages ? 2 * PAGE_SIZE : 0)
#define UNITROOM(u) ((size_t)(u->end_pc - u->start_pc))
#define UNIT_RESERVED_ROOM(u) (u->reserved_end_pc - u->start_pc)
* the committed size of the unit by adding header size to available size
*/
#define UNIT_COMMIT_SIZE(u) (UNITROOM(u) + sizeof(heap_unit_t))
#define UNIT_RESERVED_SIZE(u) (UNIT_RESERVED_ROOM(u) + sizeof(heap_unit_t))
#define UNIT_ALLOC_START(u) (u->start_pc - sizeof(heap_unit_t))
#define UNIT_GET_START_PC(u) (byte *)(((ptr_uint_t)u) + sizeof(heap_unit_t))
#define UNIT_COMMIT_END(u) (u->end_pc)
#define UNIT_RESERVED_END(u) (u->reserved_end_pc)
* as those are not considered part of the usable space).
*/
#define UNITALLOC(u) (UNIT_RESERVED_SIZE(u))
#define UNITOVERHEAD sizeof(heap_unit_t)
#define MAXROOM (HEAP_UNIT_MAX_SIZE - UNITOVERHEAD)
#define MAX_VALID_HEAP_ALLOCATION INT_MAX
* this struct is kept at top of unit itself, not in separate allocation
*/
typedef struct _heap_unit_t {
heap_pc start_pc;
heap_pc end_pc;
heap_pc cur_pc;
heap_pc reserved_end_pc;
bool in_vmarea_list;
which_vmm_t which;
#ifdef DEBUG
int id;
#endif
struct _heap_unit_t *next_local;
struct _heap_unit_t *next_global;
struct _heap_unit_t *prev_global;
} heap_unit_t;
#ifdef HEAP_ACCOUNTING
typedef struct _heap_acct_t {
size_t alloc_reuse[ACCT_LAST];
size_t alloc_new[ACCT_LAST];
size_t cur_usage[ACCT_LAST];
size_t max_usage[ACCT_LAST];
size_t max_single[ACCT_LAST];
uint num_alloc[ACCT_LAST];
} heap_acct_t;
#endif
* a heap_t is a collection of units with the same properties
* to reflect that this is used for more than just thread-private memory.
* Also rename the "tu" vars to "h"
*/
typedef struct _thread_units_t {
heap_unit_t *top_unit;
heap_unit_t *cur_unit;
heap_pc free_list[BLOCK_TYPES];
#ifdef DEBUG
int num_units;
#endif
dcontext_t *dcontext;
which_vmm_t which;
bool writable;
#ifdef HEAP_ACCOUNTING
heap_acct_t acct;
#endif
} thread_units_t;
#define REACHABLE_HEAP() (IF_X64_ELSE(DYNAMO_OPTION(reachable_heap), true))
typedef struct _thread_heap_t {
thread_units_t *local_heap;
* both to enable resetting memory and for safety for unlink flushing in the presence
* of clean calls out of the cache that might allocate IR memory (which does not
* use nonpersistent heap). Any client actions that involve fragments or linking
* should require couldbelinking status, which makes them safe wrt unlink flushing.
* Xref DrMi#1791.
*/
thread_units_t *nonpersistent_heap;
thread_units_t *reachable_heap;
#ifdef UNIX
heap_pc fork_copy_start;
size_t fork_copy_size;
vm_area_vector_t *fork_copy_areas;
#endif
} thread_heap_t;
* FIXME: give this name to thread_units_t, and name this AllHeapUnits
*/
typedef struct _heap_t {
heap_unit_t *units;
heap_unit_t *dead;
* for release build too, so it's separate...can we do better?
*/
uint num_dead;
} heap_t;
static bool heap_exiting = false;
#ifdef DEBUG
DECLARE_NEVERPROT_VAR(static bool ever_beyond_vmm, false);
#endif
* Must be recursive due to circular dependencies between vmareas and global heap.
* Furthermore, always grab dynamo_vm_areas_lock() before grabbing this lock,
* to make DR areas update and heap alloc/free atomic!
*/
DECLARE_CXTSWPROT_VAR(static recursive_lock_t heap_unit_lock,
INIT_RECURSIVE_LOCK(heap_unit_lock));
* that global_alloc_lock MUST be grabbed first, to avoid deadlocks
*/
* creation and global heap alloc
* must be recursive so that heap_vmareas_synch_units can hold it and heap_unit_lock
* up front to avoid deadlocks, and still allow vmareas to global_alloc --
* BUT we do NOT want global_heap_alloc() to be able to recurse!
* FIXME: either find a better solution to the heap_vmareas_synch_units deadlock
* that is as efficient, or find a way to assert that the only recursion is
* from heap_vmareas_synch_units to global_alloc
*/
DECLARE_CXTSWPROT_VAR(static recursive_lock_t global_alloc_lock,
INIT_RECURSIVE_LOCK(global_alloc_lock));
DECLARE_CXTSWPROT_VAR(static recursive_lock_t low_on_memory_pending_lock,
INIT_RECURSIVE_LOCK(low_on_memory_pending_lock));
DECLARE_FREQPROT_VAR(bool low_on_memory_pending, false);
#if defined(DEBUG) && defined(HEAP_ACCOUNTING) && defined(HOT_PATCHING_INTERFACE)
static int
get_special_heap_header_size(void);
#endif
vm_area_vector_t *landing_pad_areas;
#ifdef WINDOWS
static app_pc lpad_temp_writable_start;
static size_t lpad_temp_writable_size;
static void
release_landing_pad_mem(void);
#endif
* DR areas lock first, to retry
*/
static bool
safe_to_allocate_or_free_heap_units()
{
return ((!self_owns_recursive_lock(&global_alloc_lock) &&
!self_owns_recursive_lock(&heap_unit_lock)) ||
self_owns_dynamo_vm_area_lock());
}
* protected by the heap_unit_lock
*/
DECLARE_FREQPROT_VAR(static bool dynamo_areas_pending_remove, false);
#ifdef HEAP_ACCOUNTING
const char *whichheap_name[] = {
"BB Fragments",
"Coarse Links",
"Future Frag",
"Frag Tables",
"IBL Tables",
"Traces",
"FC Empties",
"Vm Multis",
"IR",
"RCT Tables",
"VM Areas",
"Symbols",
# ifdef SIDELINE
"Sideline",
# endif
"TH Counter",
"Tombstone",
"Hot Patching",
"Thread Mgt",
"Memory Mgt",
"Stats",
"SpecialHeap",
"Client",
"Lib Dup",
"Clean Call",
"Other",
};
* follow a two-pronged strategy:
* 1) For accurate stats we add a thread's final stats to the global only
* when it is cleaned up. But, this prevents global stats from being
* available in the middle of a run or if a run is not cleaned up nicely.
* 2) We have a set of heap_accounting stats for incremental global stats
* that are available at any time, yet racy and so may be off a little.
*/
DECLARE_NEVERPROT_VAR(static thread_units_t global_racy_units, { 0 });
# define ACCOUNT_FOR_ALLOC_HELPER(type, tu, which, alloc_sz, ask_sz) \
do { \
(tu)->acct.type[which] += alloc_sz; \
(tu)->acct.num_alloc[which]++; \
(tu)->acct.cur_usage[which] += alloc_sz; \
if ((tu)->acct.cur_usage[which] > (tu)->acct.max_usage[which]) \
(tu)->acct.max_usage[which] = (tu)->acct.cur_usage[which]; \
if (ask_sz > (tu)->acct.max_single[which]) \
(tu)->acct.max_single[which] = ask_sz; \
} while (0)
# define ACCOUNT_FOR_ALLOC(type, tu, which, alloc_sz, ask_sz) \
do { \
STATS_ADD_PEAK(heap_claimed, alloc_sz); \
ACCOUNT_FOR_ALLOC_HELPER(type, tu, which, alloc_sz, ask_sz); \
ACCOUNT_FOR_ALLOC_HELPER(type, &global_racy_units, which, alloc_sz, ask_sz); \
} while (0)
# define ACCOUNT_FOR_FREE(tu, which, size) \
do { \
STATS_SUB(heap_claimed, (size)); \
(tu)->acct.cur_usage[which] -= size; \
global_racy_units.acct.cur_usage[which] -= size; \
} while (0)
#else
# define ACCOUNT_FOR_ALLOC(type, tu, which, alloc_sz, ask_sz)
# define ACCOUNT_FOR_FREE(tu, which, size)
#endif
typedef byte *vm_addr_t;
#ifdef X64
static byte *heap_allowable_region_start = (byte *)PTR_UINT_0;
static byte *heap_allowable_region_end = (byte *)POINTER_MAX;
* This lets apps grow beyond 4G of heap.
*/
# define HEAP_REACHABILITY_ENABLED() (!standalone_library)
* used in request_region_be_heap_reachable().
*/
DECLARE_CXTSWPROT_VAR(static mutex_t request_region_be_heap_reachable_lock,
INIT_LOCK_FREE(request_region_be_heap_reachable_lock));
* request_region_be_heap_reachable_lock.
*/
static byte *must_reach_region_start = (byte *)POINTER_MAX;
static byte *must_reach_region_end = (byte *)PTR_UINT_0;
static void
reset_heap_reachable_bounds(void)
{
heap_allowable_region_start = (byte *)PTR_UINT_0;
heap_allowable_region_end = (byte *)POINTER_MAX;
must_reach_region_start = (byte *)POINTER_MAX;
must_reach_region_end = (byte *)PTR_UINT_0;
}
* be called before vmm_heap_init() so we can place the DR heap to meet these constraints.
* Can also be called post vmm_heap_init() but at that point acts as an assert that the
* supplied region is reachable since the heap is already reserved.
*
* Must be called at least once up front, for the -heap_in_lower_4GB code here
* to kick in!
*/
void
request_region_be_heap_reachable(byte *start, size_t size)
{
if (!HEAP_REACHABILITY_ENABLED())
return;
LOG(GLOBAL, LOG_HEAP, 2,
"Adding must-be-reachable-from-heap region " PFX "-" PFX "\n"
"Existing must-be-reachable region " PFX "-" PFX "\n"
"Existing allowed range " PFX "-" PFX "\n",
start, start + size, must_reach_region_start, must_reach_region_end,
heap_allowable_region_start, heap_allowable_region_end);
ASSERT(!POINTER_OVERFLOW_ON_ADD(start, size));
ASSERT(size > 0);
d_r_mutex_lock(&request_region_be_heap_reachable_lock);
if (start < must_reach_region_start) {
byte *allowable_end_tmp;
SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
must_reach_region_start = start;
allowable_end_tmp =
REACHABLE_32BIT_END(must_reach_region_start, must_reach_region_end);
if (DYNAMO_OPTION(heap_in_lower_4GB) &&
allowable_end_tmp > (byte *)POINTER_MAX_32BIT) {
allowable_end_tmp = (byte *)POINTER_MAX_32BIT;
}
* heap_allowable_region_end. */
heap_allowable_region_end = allowable_end_tmp;
SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
}
if (start + size - 1 > must_reach_region_end) {
SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
must_reach_region_end = start + size - 1;
* heap_allowable_region_start. */
heap_allowable_region_start =
REACHABLE_32BIT_START(must_reach_region_start, must_reach_region_end);
SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
}
ASSERT(must_reach_region_start <= must_reach_region_end);
ASSERT(!DYNAMO_OPTION(heap_in_lower_4GB) ||
heap_allowable_region_end <= (byte *)POINTER_MAX_32BIT);
d_r_mutex_unlock(&request_region_be_heap_reachable_lock);
LOG(GLOBAL, LOG_HEAP, 1,
"Added must-be-reachable-from-heap region " PFX "-" PFX "\n"
"New must-be-reachable region " PFX "-" PFX "\n"
"New allowed range " PFX "-" PFX "\n",
start, start + size, must_reach_region_start, must_reach_region_end,
heap_allowable_region_start, heap_allowable_region_end);
* control where DR/client dlls are loaded these could fire if rebased). */
ASSERT(heap_allowable_region_start <= must_reach_region_start &&
"x64 reachability contraints not satisfiable");
ASSERT(must_reach_region_end <= heap_allowable_region_end &&
"x64 reachability contraints not satisfiable");
if (heap_allowable_region_start > must_reach_region_start ||
must_reach_region_end > heap_allowable_region_end) {
* drastic than triggering a FATAL_USAGE_ERROR. */
FATAL_USAGE_ERROR(HEAP_CONTRAINTS_UNSATISFIABLE, 2, get_application_name(),
get_application_pid());
}
}
void
vmcode_get_reachable_region(byte **region_start OUT, byte **region_end OUT)
{
* align when asked about it.
*/
if (region_start != NULL)
*region_start = (byte *)ALIGN_FORWARD(heap_allowable_region_start, PAGE_SIZE);
if (region_end != NULL)
*region_end = (byte *)ALIGN_BACKWARD(heap_allowable_region_end, PAGE_SIZE);
}
#endif
static void
threadunits_init(dcontext_t *dcontext, thread_units_t *tu, size_t size, bool reachable);
static void
threadunits_exit(thread_units_t *tu, dcontext_t *dcontext);
static void *
common_heap_alloc(thread_units_t *tu, size_t size HEAPACCT(which_heap_t which));
static bool
common_heap_free(thread_units_t *tu, void *p, size_t size HEAPACCT(which_heap_t which));
static void
release_real_memory(void *p, size_t size, bool remove_vm, which_vmm_t which);
static void
release_guarded_real_memory(vm_addr_t p, size_t size, bool remove_vm, bool guarded,
which_vmm_t which);
typedef enum {
* check for incompatible kernel drivers
*/
OOM_INIT = 0x1,
* increase -vm_size to reserve more memory
*/
OOM_RESERVE = 0x2,
* Increase pagefile size, check for memory leak in any application.
*
* FIXME: possible automatic actions
* if systemwide failure we may want to wait if transient
* FIXME: if in a job latter we want to detect and just die
* (though after freeing as much memory as we can)
*/
OOM_COMMIT = 0x4,
* as a possible workaround increasing -heap_commit_increment
* may make expose us to commit-ing less frequently,
* On the other hand committing smaller chunks has a higher
* chance of getting through when there is very little memory.
*
* FIXME: not much more informative than OOM_COMMIT
*/
OOM_EXTEND = 0x8,
} oom_source_t;
static void
report_low_on_memory(which_vmm_t which, oom_source_t source,
heap_error_code_t os_error_code);
#define MAX_VMCODE_SIZE (2ULL * 1024 * 1024 * 1024)
#define MAX_VMHEAP_SIZE (IF_X64_ELSE(128ULL, (4ULL - 1)) * 1024 * 1024 * 1024)
* the maximum we should count on in one process
*/
#define MIN_VMM_HEAP_UNIT_SIZE DYNAMO_OPTION(vmm_block_size)
typedef struct {
vm_addr_t start_addr;
vm_addr_t end_addr;
vm_addr_t alloc_start;
size_t alloc_size;
* if so must update the bitmap_t routines
*/
uint num_blocks;
mutex_t lock;
static therefore we don't grab locks on read accesses. Anyways,
currently the bitmap_t is used with no write intent only for ASSERTs. */
uint num_free_blocks;
const char *name;
* vmcode and vmheap and to allow for large vmheap sizes.
* We place it at start_addr, or the writable equivalent for vmcode.
*/
bitmap_element_t *blocks;
} vm_heap_t;
* Note that we do have static structs for bootstrapping and we later move
* the data here.
*/
typedef struct _heap_management_t {
* code cache and thus is called "vmcode", from our heap which can go anywhere,
* "vmheap".
* For each, we reserve a single vm_heap_t for guaranteed allocation.
* We fall back to the OS when run out of reservation space.
* If REACHABLE_HEAP() we do not use vmheap and put everything in
* vmcode.
*/
vm_heap_t vmheap;
* be larger than 2G anyway.
* XXX i#1132: for 64-bit, we make it 2G by default so we never have to
* fall back to the OS! We'll have to make room to load client libs inside
* the VMM-managed space is all.
* For 32-bit it will have to remain smaller and handle falling back to the OS.
*/
vm_heap_t vmcode;
file_t dual_map_file;
vm_addr_t vmcode_writable_base;
vm_addr_t vmcode_writable_alloc;
heap_t heap;
thread_units_t global_units;
thread_units_t global_nonpersistent_units;
bool global_heap_writable;
thread_units_t global_unprotected_units;
thread_units_t global_reachable_units;
} heap_management_t;
* temp_heapmgt.lock is initialized in vmm_heap_unit_init().
*/
static heap_management_t temp_heapmgt;
static heap_management_t *heapmgt = &temp_heapmgt;
static bool vmm_heap_exited = false;
should change the interface for the last stack
*/
#define MEMORY_FILE_NAME "dynamorio_dual_map"
static vm_addr_t
vmm_heap_reserve_blocks(vm_heap_t *vmh, size_t size_in, byte *base, which_vmm_t which);
static bool
vmm_heap_commit(vm_addr_t p, size_t size, uint prot, heap_error_code_t *error_code,
which_vmm_t which);
static inline uint
vmm_addr_to_block(vm_heap_t *vmh, vm_addr_t p)
{
ASSERT(
CHECK_TRUNCATE_TYPE_uint((p - vmh->start_addr) / DYNAMO_OPTION(vmm_block_size)));
return (uint)((p - vmh->start_addr) / DYNAMO_OPTION(vmm_block_size));
}
static inline vm_addr_t
vmm_block_to_addr(vm_heap_t *vmh, uint block)
{
ASSERT(block >= 0 && block < vmh->num_blocks);
return (vm_addr_t)(vmh->start_addr + block * DYNAMO_OPTION(vmm_block_size));
}
static bool
vmm_in_same_block(vm_heap_t *vmh, vm_addr_t p1, vm_addr_t p2)
{
return vmm_addr_to_block(vmh, p1) == vmm_addr_to_block(vmh, p2);
}
#if defined(DEBUG) && defined(INTERNAL)
static void
vmm_dump_map(vm_heap_t *vmh)
{
uint i;
bitmap_element_t *b = vmh->blocks;
uint bitmap_size = vmh->num_blocks;
uint last_i = 0;
bool is_used = bitmap_test(b, 0) == 0;
LOG(GLOBAL, LOG_HEAP, 3, "vmm_dump_map(" PFX ")\n", vmh);
* too big. We disable but leave the capability to enable one-off use.
*/
DOLOG(20, LOG_HEAP, {
dump_buffer_as_bytes(GLOBAL, b,
BITMAP_INDEX(bitmap_size) * sizeof(bitmap_element_t),
DUMP_RAW | DUMP_ADDRESS);
});
LOG(GLOBAL, LOG_HEAP, 1, "\nvmm_dump_map(" PFX ") virtual regions\n", vmh);
# define VMM_DUMP_MAP_LOG(i, last_i) \
LOG(GLOBAL, LOG_HEAP, 1, PFX "-" PFX " size=%d %s\n", \
vmm_block_to_addr(vmh, last_i), \
vmm_block_to_addr(vmh, i - 1) + DYNAMO_OPTION(vmm_block_size) - 1, \
(i - last_i) * DYNAMO_OPTION(vmm_block_size), \
is_used ? "reserved" : "free");
for (i = 0; i < bitmap_size; i++) {
if (is_used != (bitmap_test(b, i) == 0)) {
VMM_DUMP_MAP_LOG(i, last_i);
is_used = (bitmap_test(b, i) == 0);
last_i = i;
}
}
VMM_DUMP_MAP_LOG(bitmap_size, last_i);
}
#endif
static inline void
print_vmh_data(vm_heap_t *vmh, file_t outf)
{
d_r_mutex_lock(&vmh->lock);
print_file(outf, "VM heap: addr range " PFX "--" PFX ", # free blocks %d\n",
vmh->start_addr, vmh->end_addr, vmh->num_free_blocks);
d_r_mutex_unlock(&vmh->lock);
}
void
print_vmm_heap_data(file_t outf)
{
if (heapmgt->vmheap.start_addr != NULL)
print_vmh_data(&heapmgt->vmheap, outf);
if (heapmgt->vmcode.start_addr != NULL)
print_vmh_data(&heapmgt->vmcode, outf);
}
static inline void
vmm_heap_initialize_unusable(vm_heap_t *vmh)
{
vmh->start_addr = vmh->end_addr = NULL;
vmh->num_free_blocks = vmh->num_blocks = 0;
}
static void
report_w_xor_x_fatal_error_and_exit(void)
{
REPORT_FATAL_ERROR_AND_EXIT(FAILED_TO_SATISFY_W_XOR_X, 2, get_application_name(),
get_application_pid());
ASSERT_NOT_REACHED();
}
static void
vmm_place_vmcode(vm_heap_t *vmh, size_t size, heap_error_code_t *error_code)
{
ptr_uint_t preferred = 0;
#ifdef X64
* Next comes -vm_base_near_app. It will fail for -vm_size=2G, which we document.
*/
if (DYNAMO_OPTION(vm_base_near_app)) {
* Non-static: still a good idea for fewer rip-rel manglings.
* Asking for app base means we'll prefer before the app, which
* has less of an impact on its heap.
*/
app_pc app_base = get_application_base();
app_pc app_end = get_application_end();
* code if the app base is near -vm_base.
*/
if (!REL32_REACHABLE(app_base, (app_pc)DYNAMO_OPTION(vm_base)) ||
!REL32_REACHABLE(app_base,
(app_pc)DYNAMO_OPTION(vm_base) +
DYNAMO_OPTION(vm_max_offset)) ||
((app_pc)DYNAMO_OPTION(vm_base) < app_end &&
(app_pc)DYNAMO_OPTION(vm_base) + DYNAMO_OPTION(vm_max_offset) > app_base)) {
byte *reach_base = MAX(REACHABLE_32BIT_START(app_base, app_end),
heap_allowable_region_start);
byte *reach_end =
MIN(REACHABLE_32BIT_END(app_base, app_end), heap_allowable_region_end);
if (reach_base < reach_end) {
size_t add_for_align = DYNAMO_OPTION(vmm_block_size);
if (DYNAMO_OPTION(vmm_block_size) == PAGE_SIZE) {
add_for_align = 0;
}
vmh->alloc_start = os_heap_reserve_in_region(
(void *)ALIGN_FORWARD(reach_base, PAGE_SIZE),
(void *)ALIGN_BACKWARD(reach_end, PAGE_SIZE), size + add_for_align,
error_code, true );
if (vmh->alloc_start != NULL) {
vmh->start_addr = (heap_pc)ALIGN_FORWARD(
vmh->alloc_start, DYNAMO_OPTION(vmm_block_size));
if (add_for_align == 0) {
ASSERT(ALIGNED(vmh->alloc_start, DYNAMO_OPTION(vmm_block_size)));
ASSERT(vmh->start_addr == vmh->alloc_start);
}
request_region_be_heap_reachable(app_base, app_end - app_base);
LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_unit_init: placed %s near app\n",
vmh->name);
}
}
}
}
#endif
if (vmh->start_addr == NULL) {
* since its allocation base is 64KB, and if we want to stay
* safely in say 0x20000000-0x2fffffff we're left with only 12
* bits of randomness - which may be too little. On the other
* hand changing any of the lower 16 bits will make our bugs
* non-deterministic. */
preferred = (DYNAMO_OPTION(vm_base) +
get_random_offset(DYNAMO_OPTION(vm_max_offset) /
DYNAMO_OPTION(vmm_block_size)) *
DYNAMO_OPTION(vmm_block_size));
preferred = ALIGN_FORWARD(preferred, OS_ALLOC_GRANULARITY);
ASSERT(!POINTER_OVERFLOW_ON_ADD(preferred, size));
#ifdef X64
if ((byte *)preferred < heap_allowable_region_start ||
(byte *)preferred + size > heap_allowable_region_end) {
*error_code = HEAP_ERROR_NOT_AT_PREFERRED;
LOG(GLOBAL, LOG_HEAP, 1,
"vmm_heap_unit_init preferred=" PFX " too far from " PFX "-" PFX "\n",
preferred, heap_allowable_region_start, heap_allowable_region_end);
} else {
#endif
vmh->alloc_start =
os_heap_reserve((void *)preferred, size, error_code, true );
vmh->start_addr = vmh->alloc_start;
LOG(GLOBAL, LOG_HEAP, 1,
"vmm_heap_unit_init preferred=" PFX " got start_addr=" PFX "\n",
preferred, vmh->start_addr);
#ifdef X64
}
#endif
}
while (vmh->start_addr == NULL && DYNAMO_OPTION(vm_allow_not_at_base)) {
* syslog or assert here
*/
vmh->alloc_size = size + DYNAMO_OPTION(vmm_block_size);
#ifdef X64
vmh->alloc_start = os_heap_reserve_in_region(
(void *)ALIGN_FORWARD(heap_allowable_region_start, PAGE_SIZE),
(void *)ALIGN_BACKWARD(heap_allowable_region_end, PAGE_SIZE),
size + DYNAMO_OPTION(vmm_block_size), error_code, true );
#else
vmh->alloc_start = (heap_pc)os_heap_reserve(
NULL, size + DYNAMO_OPTION(vmm_block_size), error_code, true );
#endif
vmh->start_addr =
(heap_pc)ALIGN_FORWARD(vmh->alloc_start, DYNAMO_OPTION(vmm_block_size));
LOG(GLOBAL, LOG_HEAP, 1,
"vmm_heap_unit_init unable to allocate at preferred=" PFX
" letting OS place sz=%dM addr=" PFX "\n",
preferred, size / (1024 * 1024), vmh->start_addr);
if (vmh->alloc_start == NULL && DYNAMO_OPTION(vm_allow_smaller)) {
size_t sub = (size_t)ALIGN_FORWARD(size / 16, 1024 * 1024);
SYSLOG_INTERNAL_WARNING_ONCE("Full size vmm heap allocation failed");
if (size > sub)
size -= sub;
else
break;
} else
break;
}
#ifdef X64
if (DYNAMO_OPTION(satisfy_w_xor_x)) {
* whose MAP_FILE_REACHABLE relies on VMM (us!) being initialized, which is
* tricky, we simply do the standard reserve above and then map our file
* on top. TODO i#3566: We need a different strategy on Windows.
*/
ASSERT(!is_vmm_reserved_address(vmh->start_addr, size, NULL, NULL));
size_t map_size = vmh->alloc_size;
byte *map_base =
os_map_file(heapmgt->dual_map_file, &map_size, 0, vmh->alloc_start,
MEMPROT_NONE, MAP_FILE_VMM_COMMIT | MAP_FILE_FIXED);
if (map_base != vmh->alloc_start || map_size != vmh->alloc_size) {
report_w_xor_x_fatal_error_and_exit();
ASSERT_NOT_REACHED();
}
}
if (vmh->start_addr != NULL) {
ASSERT(vmh->start_addr >= heap_allowable_region_start &&
!POINTER_OVERFLOW_ON_ADD(vmh->start_addr, size) &&
vmh->start_addr + size <= heap_allowable_region_end);
request_region_be_heap_reachable(vmh->start_addr, size);
}
#endif
ASSERT(ALIGNED(vmh->start_addr, DYNAMO_OPTION(vmm_block_size)));
}
static void
vmm_heap_unit_init_failed(vm_heap_t *vmh, heap_error_code_t error_code, const char *name)
{
LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_unit_init %s: failed to allocate memory!\n", name);
vmm_heap_initialize_unusable(vmh);
report_low_on_memory(VMM_HEAP, OOM_INIT, error_code);
ASSERT_NOT_REACHED();
}
static void
vmm_heap_unit_init(vm_heap_t *vmh, size_t size, bool is_vmcode, const char *name)
{
heap_error_code_t error_code = 0;
ASSIGN_INIT_LOCK_FREE(vmh->lock, vmh_lock);
* temp_heapmgt, else it will point to freed memory when we go back to temp_heapmgt
* for lock cleanup code.
*/
d_r_mutex_lock(&vmh->lock);
d_r_mutex_unlock(&vmh->lock);
size = ALIGN_FORWARD(size, DYNAMO_OPTION(vmm_block_size));
vmh->alloc_size = size;
vmh->start_addr = NULL;
vmh->name = name;
if (size == 0) {
vmm_heap_initialize_unusable(vmh);
return;
}
if (is_vmcode) {
* controlled by runtime options.
*/
if (DYNAMO_OPTION(satisfy_w_xor_x)) {
heapmgt->dual_map_file = os_create_memory_file(MEMORY_FILE_NAME, size);
if (heapmgt->dual_map_file == INVALID_FILE) {
report_w_xor_x_fatal_error_and_exit();
ASSERT_NOT_REACHED();
}
}
vmm_place_vmcode(vmh, size, &error_code);
if (DYNAMO_OPTION(satisfy_w_xor_x)) {
size_t map_size = vmh->alloc_size;
heapmgt->vmcode_writable_alloc =
os_map_file(heapmgt->dual_map_file, &map_size, 0, NULL, MEMPROT_NONE, 0);
ASSERT(map_size == vmh->alloc_size);
if (heapmgt->vmcode_writable_alloc == 0) {
LOG(GLOBAL, LOG_HEAP, 1,
"vmm_heap_unit_init %s: failed to allocate writable vmcode!\n");
vmm_heap_initialize_unusable(vmh);
report_low_on_memory(VMM_CACHE | VMM_REACHABLE, OOM_INIT, error_code);
ASSERT_NOT_REACHED();
}
heapmgt->vmcode_writable_base = (heap_pc)ALIGN_FORWARD(
heapmgt->vmcode_writable_alloc, DYNAMO_OPTION(vmm_block_size));
LOG(GLOBAL, LOG_HEAP, 1,
"vmm_heap_unit_init vmcode+w reservation: [" PFX "," PFX ")\n",
heapmgt->vmcode_writable_base, heapmgt->vmcode_writable_base + size);
}
} else {
* for this second reservation and rely on the OS.
*/
vmh->alloc_size = size + DYNAMO_OPTION(vmm_block_size);
vmh->alloc_start = (heap_pc)os_heap_reserve(
NULL, size + DYNAMO_OPTION(vmm_block_size), &error_code, false );
vmh->start_addr =
(heap_pc)ALIGN_FORWARD(vmh->alloc_start, DYNAMO_OPTION(vmm_block_size));
}
if (vmh->start_addr == 0) {
vmm_heap_unit_init_failed(vmh, error_code, name);
ASSERT_NOT_REACHED();
}
vmh->end_addr = vmh->start_addr + size;
ASSERT_TRUNCATE(vmh->num_blocks, uint, size / DYNAMO_OPTION(vmm_block_size));
vmh->num_blocks = (uint)(size / DYNAMO_OPTION(vmm_block_size));
size_t blocks_sz_bytes = BITMAP_INDEX(vmh->num_blocks) * sizeof(bitmap_element_t);
blocks_sz_bytes = ALIGN_FORWARD(blocks_sz_bytes, DYNAMO_OPTION(vmm_block_size));
vmh->blocks = (bitmap_element_t *)vmh->start_addr;
if (is_vmcode)
vmh->blocks = (bitmap_element_t *)vmcode_get_writable_addr((byte *)vmh->blocks);
vmh->num_free_blocks = vmh->num_blocks;
LOG(GLOBAL, LOG_HEAP, 1,
"vmm_heap_unit_init %s reservation: [" PFX "," PFX ") total=%d free=%d\n", name,
vmh->start_addr, vmh->end_addr, vmh->num_blocks, vmh->num_free_blocks);
* The size was aligned above.
*/
ASSERT(ALIGNED(vmh->blocks, DYNAMO_OPTION(vmm_block_size)));
which_vmm_t which = VMM_HEAP | (is_vmcode ? VMM_REACHABLE : 0);
vmm_heap_commit((vm_addr_t)vmh->blocks, blocks_sz_bytes, MEMPROT_READ | MEMPROT_WRITE,
&error_code, which);
if (error_code != 0) {
vmm_heap_unit_init_failed(vmh, error_code, name);
ASSERT_NOT_REACHED();
}
bitmap_initialize_free(vmh->blocks, vmh->num_blocks);
vmm_heap_reserve_blocks(vmh, blocks_sz_bytes, vmh->start_addr, which);
DOLOG(1, LOG_HEAP, { vmm_dump_map(vmh); });
ASSERT(bitmap_check_consistency(vmh->blocks, vmh->num_blocks, vmh->num_free_blocks));
}
static void
vmm_heap_unit_exit(vm_heap_t *vmh)
{
LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_unit_exit %s [" PFX "," PFX ") total=%d free=%d\n",
vmh->name, vmh->start_addr, vmh->end_addr, vmh->num_blocks, vmh->num_free_blocks);
DELETE_LOCK(vmh->lock);
if (vmh->start_addr == NULL)
return;
DOLOG(1, LOG_HEAP, { vmm_dump_map(vmh); });
ASSERT(bitmap_check_consistency(vmh->blocks, vmh->num_blocks, vmh->num_free_blocks));
ASSERT(vmh->num_blocks * DYNAMO_OPTION(vmm_block_size) ==
(ptr_uint_t)(vmh->end_addr - vmh->start_addr));
* that is what we'll do, otherwise it will stay up forever.
*/
bool free_heap = vmh->num_free_blocks == vmh->num_blocks;
#ifdef UNIX
* doing a detach we can be sure our stack is not actually in the heap.
*/
if (doing_detach) {
DODEBUG({
byte *sp;
GET_STACK_PTR(sp);
ASSERT(!(sp >= vmh->start_addr && sp < vmh->end_addr));
});
free_heap = true;
}
#endif
if (free_heap) {
heap_error_code_t error_code;
os_heap_free(vmh->alloc_start, vmh->alloc_size, &error_code);
ASSERT(error_code == HEAP_ERROR_SUCCESS);
if (DYNAMO_OPTION(satisfy_w_xor_x) && vmh == &heapmgt->vmcode) {
os_heap_free(heapmgt->vmcode_writable_alloc, vmh->alloc_size, &error_code);
ASSERT(error_code == HEAP_ERROR_SUCCESS);
os_delete_memory_file(MEMORY_FILE_NAME, heapmgt->dual_map_file);
heapmgt->dual_map_file = INVALID_FILE;
}
} else {
* detach scenarios where we should try to clean up from the
* virtual address space
*/
}
vmm_heap_initialize_unusable(vmh);
}
* out internally via our vm_heap_t; asserts that the address was also
* logically reserved within the vm_heap_t.
*/
static bool
vmm_is_reserved_unit(vm_heap_t *vmh, vm_addr_t p, size_t size)
{
size = ALIGN_FORWARD(size, DYNAMO_OPTION(vmm_block_size));
if (p < vmh->start_addr || vmh->end_addr < p ||
vmh->end_addr < (p + size))
return false;
ASSERT(CHECK_TRUNCATE_TYPE_uint(size / DYNAMO_OPTION(vmm_block_size)));
ASSERT(bitmap_are_reserved_blocks(vmh->blocks, vmh->num_blocks,
vmm_addr_to_block(vmh, p),
(uint)(size / DYNAMO_OPTION(vmm_block_size))));
return true;
}
static inline bool
is_vmh_reserved_address(vm_heap_t *vmh, byte *pc, size_t size, OUT byte **region_start,
OUT byte **region_end)
{
* assert, which we want to maintain for callers only dealing with
* DR-allocated addresses, while this routine is called w/ random
* addresses
*/
if (pc >= vmh->start_addr && !POINTER_OVERFLOW_ON_ADD(pc, size) &&
(pc + size) <= vmh->end_addr) {
if (region_start != NULL)
*region_start = vmh->start_addr;
if (region_end != NULL)
*region_end = vmh->end_addr;
return true;
}
return false;
}
* out internally via our vm_heap_t. Optionally returns the bounds of the region.
* Does not consider memory we allocate once we run out of our original reservations.
*/
bool
is_vmm_reserved_address(byte *pc, size_t size, OUT byte **region_start,
OUT byte **region_end)
{
ASSERT(heapmgt != NULL);
if (heapmgt->vmheap.start_addr != NULL &&
is_vmh_reserved_address(&heapmgt->vmheap, pc, size, region_start, region_end))
return true;
if (heapmgt->vmcode.start_addr != NULL &&
is_vmh_reserved_address(&heapmgt->vmcode, pc, size, region_start, region_end))
return true;
if (heapmgt->vmcode_writable_base != NULL &&
is_vmh_reserved_address(&heapmgt->vmcode, vmcode_get_executable_addr(pc), size,
region_start, region_end)) {
if (region_start != NULL)
*region_start = vmcode_get_writable_addr(*region_start);
if (region_end != NULL)
*region_end = vmcode_get_writable_addr(*region_end);
return true;
}
return false;
}
byte *
vmcode_get_start(void)
{
if (heapmgt->vmcode.start_addr != NULL)
return heapmgt->vmcode.start_addr;
if (heapmgt->vmheap.start_addr != NULL)
return heapmgt->vmheap.start_addr;
return NULL;
}
byte *
vmcode_get_end(void)
{
if (heapmgt->vmcode.start_addr != NULL)
return heapmgt->vmcode.end_addr;
if (heapmgt->vmheap.start_addr != NULL)
return heapmgt->vmheap.end_addr;
return NULL;
}
static vm_heap_t *
vmheap_for_which(which_vmm_t which)
{
if (TEST(VMM_REACHABLE, which) || REACHABLE_HEAP())
return &heapmgt->vmcode;
else
return &heapmgt->vmheap;
}
byte *
vmcode_get_writable_addr(byte *exec_addr)
{
* at the most efficient level, and are always properly paired.
*/
PTHREAD_JIT_WRITE();
if (!DYNAMO_OPTION(satisfy_w_xor_x))
return exec_addr;
* to change things like set_selfmod_sandbox_offsets()'s call to
* encode_with_patch_list() into a stack buffer.
*/
if (exec_addr < heapmgt->vmcode.start_addr || exec_addr >= heapmgt->vmcode.end_addr)
return exec_addr;
return (exec_addr - heapmgt->vmcode.start_addr) + heapmgt->vmcode_writable_base;
}
byte *
vmcode_get_executable_addr(byte *write_addr)
{
if (!DYNAMO_OPTION(satisfy_w_xor_x))
return write_addr;
if (write_addr < heapmgt->vmcode_writable_base ||
write_addr >= heapmgt->vmcode_writable_base +
(heapmgt->vmcode.end_addr - heapmgt->vmcode.start_addr))
return write_addr;
return (write_addr - heapmgt->vmcode_writable_base) + heapmgt->vmcode.start_addr;
}
#ifdef DEBUG_MEMORY
static inline byte *
vmm_get_writable_addr(byte *exec_addr, which_vmm_t which)
{
vm_heap_t *vmh = vmheap_for_which(which);
if (vmh == &heapmgt->vmcode)
return vmcode_get_writable_addr(exec_addr);
return exec_addr;
}
#endif
static inline vm_addr_t
vmm_normalize_addr(vm_heap_t *vmh, INOUT vm_addr_t *p_exec)
{
vm_addr_t p = *p_exec;
if (p < vmh->start_addr || p >= vmh->end_addr) {
p = (p - heapmgt->vmcode_writable_base) + vmh->start_addr;
*p_exec = p;
}
return (p - vmh->start_addr) + heapmgt->vmcode_writable_base;
}
#ifdef WINDOWS
static byte *
vmheap_get_start(void)
{
if (heapmgt->vmheap.start_addr != NULL)
return heapmgt->vmheap.start_addr;
if (heapmgt->vmcode.start_addr != NULL)
return heapmgt->vmcode.start_addr;
return NULL;
}
#endif
static inline bool
has_guard_pages(which_vmm_t which)
{
if (!DYNAMO_OPTION(guard_pages))
return false;
if (TEST(VMM_PER_THREAD, which) && !DYNAMO_OPTION(per_thread_guard_pages))
return false;
return true;
}
void
iterate_vmm_regions(void (*cb)(byte *region_start, byte *region_end, void *user_data),
void *user_data)
{
if (heapmgt->vmcode.start_addr != NULL)
(*cb)(heapmgt->vmcode.start_addr, heapmgt->vmcode.end_addr, user_data);
if (heapmgt->vmheap.start_addr != NULL)
(*cb)(heapmgt->vmheap.start_addr, heapmgt->vmheap.end_addr, user_data);
if (heapmgt->vmcode_writable_base != NULL) {
(*cb)(heapmgt->vmcode_writable_base,
heapmgt->vmcode_writable_base +
(heapmgt->vmcode.end_addr - heapmgt->vmcode.start_addr),
user_data);
}
}
byte *
vmcode_unreachable_pc(void)
{
#ifdef X86_64
* for DR_CLEANCALL_INDIRECT, so ideally we want to not just provide an
* address that vmcode can't reach.
* We use a non-canonical address for x86_64.
*/
return (byte *)0x8000000100000000ULL;
#else
ptr_uint_t start = (ptr_uint_t)vmcode_get_start();
ptr_uint_t end = (ptr_uint_t)vmcode_get_end();
if (start > INT_MAX)
return NULL;
else {
return (byte *)end + INT_MAX + PAGE_SIZE;
}
#endif
}
bool
rel32_reachable_from_vmcode(byte *tgt)
{
#ifdef X64
* heap range and not just the vmcode range (i#1479).
*/
ptr_int_t new_offs = (tgt > heap_allowable_region_start)
? (tgt - heap_allowable_region_start)
: (heap_allowable_region_end - tgt);
ASSERT(vmcode_get_start() >= heap_allowable_region_start ||
!DYNAMO_OPTION(vm_reserve));
ASSERT(vmcode_get_end() <= heap_allowable_region_end + 1 ||
!DYNAMO_OPTION(vm_reserve));
return REL32_REACHABLE_OFFS(new_offs);
#else
return true;
#endif
}
bool
rel32_reachable_from_current_vmcode(byte *tgt)
{
#ifdef X64
ptr_int_t new_offs = (tgt > must_reach_region_start) ? (tgt - must_reach_region_start)
: (must_reach_region_end - tgt);
return REL32_REACHABLE_OFFS(new_offs);
#else
return true;
#endif
}
static inline void
vmm_update_block_stats(which_vmm_t which, uint num_blocks, bool add)
{
* We confirm our assumptions here.
*/
ASSERT(!TESTALL(VMM_REACHABLE | VMM_STACK, which) &&
(TEST(VMM_REACHABLE, which) || !TEST(VMM_CACHE, which)));
if (add) {
if (TEST(VMM_HEAP, which)) {
if (TEST(VMM_REACHABLE, which))
RSTATS_ADD_PEAK(vmm_blocks_reach_heap, num_blocks);
else
RSTATS_ADD_PEAK(vmm_blocks_unreach_heap, num_blocks);
} else if (TEST(VMM_CACHE, which))
RSTATS_ADD_PEAK(vmm_blocks_reach_cache, num_blocks);
else if (TEST(VMM_STACK, which))
RSTATS_ADD_PEAK(vmm_blocks_unreach_stack, num_blocks);
else if (TEST(VMM_SPECIAL_HEAP, which)) {
if (TEST(VMM_REACHABLE, which))
RSTATS_ADD_PEAK(vmm_blocks_reach_special_heap, num_blocks);
else
RSTATS_ADD_PEAK(vmm_blocks_unreach_special_heap, num_blocks);
} else if (TEST(VMM_SPECIAL_MMAP, which)) {
if (TEST(VMM_REACHABLE, which))
RSTATS_ADD_PEAK(vmm_blocks_reach_special_mmap, num_blocks);
else
RSTATS_ADD_PEAK(vmm_blocks_unreach_special_mmap, num_blocks);
}
} else {
if (TEST(VMM_HEAP, which)) {
if (TEST(VMM_REACHABLE, which))
RSTATS_SUB(vmm_blocks_reach_heap, num_blocks);
else
RSTATS_SUB(vmm_blocks_unreach_heap, num_blocks);
} else if (TEST(VMM_CACHE, which))
RSTATS_SUB(vmm_blocks_reach_cache, num_blocks);
else if (TEST(VMM_STACK, which))
RSTATS_SUB(vmm_blocks_unreach_stack, num_blocks);
else if (TEST(VMM_SPECIAL_HEAP, which)) {
if (TEST(VMM_REACHABLE, which))
RSTATS_SUB(vmm_blocks_reach_special_heap, num_blocks);
else
RSTATS_SUB(vmm_blocks_unreach_special_heap, num_blocks);
} else if (TEST(VMM_SPECIAL_MMAP, which)) {
if (TEST(VMM_REACHABLE, which))
RSTATS_SUB(vmm_blocks_reach_special_mmap, num_blocks);
else
RSTATS_SUB(vmm_blocks_unreach_special_mmap, num_blocks);
}
}
}
* (e.g. 64KB) but the caller is not forced to request at that
* alignment. We explicitly synchronize reservations and decommits
* within the vm_heap_t.
* Returns NULL if the VMMHeap is full or too fragmented to satisfy
* the request.
*/
static vm_addr_t
vmm_heap_reserve_blocks(vm_heap_t *vmh, size_t size_in, byte *base, which_vmm_t which)
{
vm_addr_t p;
uint request;
uint first_block;
size_t size;
uint must_start;
size = ALIGN_FORWARD(size_in, DYNAMO_OPTION(vmm_block_size));
ASSERT_TRUNCATE(request, uint, size / DYNAMO_OPTION(vmm_block_size));
request = (uint)(size / DYNAMO_OPTION(vmm_block_size));
if (base != NULL)
must_start = vmm_addr_to_block(vmh, base);
else
must_start = UINT_MAX;
LOG(GLOBAL, LOG_HEAP, 2,
"vmm_heap_reserve_blocks %s: size=%d => %d in blocks=%d free_blocks=%d\n",
vmh->name, size_in, size, request, vmh->num_free_blocks);
d_r_mutex_lock(&vmh->lock);
if (vmh->num_free_blocks < request) {
d_r_mutex_unlock(&vmh->lock);
return NULL;
}
first_block =
bitmap_allocate_blocks(vmh->blocks, vmh->num_blocks, request, must_start);
if (first_block != BITMAP_NOT_FOUND) {
vmh->num_free_blocks -= request;
}
d_r_mutex_unlock(&vmh->lock);
if (first_block != BITMAP_NOT_FOUND) {
p = vmm_block_to_addr(vmh, first_block);
RSTATS_ADD_PEAK(vmm_vsize_used, size);
STATS_ADD_PEAK(vmm_vsize_blocks_used, request);
STATS_ADD_PEAK(vmm_vsize_wasted, size - size_in);
vmm_update_block_stats(which, request, true );
DOSTATS({
if (request > 1) {
STATS_INC(vmm_multi_block_allocs);
STATS_ADD(vmm_multi_blocks, request);
}
});
} else {
p = NULL;
}
LOG(GLOBAL, LOG_HEAP, 2,
"vmm_heap_reserve_blocks %s: size=%d blocks=%d p=" PFX " index=%u\n", vmh->name,
size, request, p, first_block);
DOLOG(5, LOG_HEAP, { vmm_dump_map(vmh); });
return p;
}
* Update bookkeeping information about the freed region.
*/
static void
vmm_heap_free_blocks(vm_heap_t *vmh, vm_addr_t p, size_t size_in, which_vmm_t which)
{
uint first_block = vmm_addr_to_block(vmh, p);
uint request;
size_t size;
size = ALIGN_FORWARD(size_in, DYNAMO_OPTION(vmm_block_size));
ASSERT_TRUNCATE(request, uint, size / DYNAMO_OPTION(vmm_block_size));
request = (uint)(size / DYNAMO_OPTION(vmm_block_size));
LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_free_blocks %s: size=%d blocks=%d p=" PFX "\n",
vmh->name, size, request, p);
d_r_mutex_lock(&vmh->lock);
bitmap_free_blocks(vmh->blocks, vmh->num_blocks, first_block, request);
vmh->num_free_blocks += request;
d_r_mutex_unlock(&vmh->lock);
ASSERT(vmh->num_free_blocks <= vmh->num_blocks);
RSTATS_SUB(vmm_vsize_used, size);
STATS_SUB(vmm_vsize_blocks_used, request);
vmm_update_block_stats(which, request, false );
STATS_SUB(vmm_vsize_wasted, size - size_in);
}
* protection changes */
START_DATA_SECTION(FREQ_PROTECTED_SECTION, "w");
static bool
at_reset_at_vmm_limit(vm_heap_t *vmh)
{
return (DYNAMO_OPTION(reset_at_vmm_percent_free_limit) != 0 &&
100 * vmh->num_free_blocks <
DYNAMO_OPTION(reset_at_vmm_percent_free_limit) * vmh->num_blocks) ||
(DYNAMO_OPTION(reset_at_vmm_free_limit) != 0 &&
vmh->num_free_blocks * DYNAMO_OPTION(vmm_block_size) <
DYNAMO_OPTION(reset_at_vmm_free_limit));
}
static void
reached_beyond_vmm(which_vmm_t which)
{
DODEBUG(ever_beyond_vmm = true;);
if (INTERNAL_OPTION(rstats_to_stderr))
dump_global_rstats_to_stderr();
char message[256];
if (DYNAMO_OPTION(satisfy_w_xor_x) &&
(TEST(VMM_REACHABLE, which) || REACHABLE_HEAP())) {
* should set -vm_size 2G instead and take the rip-rel mangling hit
* (see i#3570).
*/
snprintf(
message, BUFFER_SIZE_ELEMENTS(message),
"Alloc type: 0x%x. -satisfy_w_xor_x requires VMM memory: try '-vm_size 2G'",
which);
NULL_TERMINATE_BUFFER(message);
REPORT_FATAL_ERROR_AND_EXIT(OUT_OF_VMM_CANNOT_USE_OS, 3, get_application_name(),
get_application_pid(), message);
ASSERT_NOT_REACHED();
} else {
snprintf(message, BUFFER_SIZE_ELEMENTS(message), "Alloc type: 0x%x.", which);
NULL_TERMINATE_BUFFER(message);
SYSLOG(SYSLOG_WARNING, OUT_OF_VMM_CANNOT_USE_OS, 3, get_application_name(),
get_application_pid(), message);
}
}
void
vmm_heap_handle_pending_low_on_memory_event_trigger()
{
bool trigger = false;
acquire_recursive_lock(&low_on_memory_pending_lock);
if (low_on_memory_pending) {
bool value = false;
ATOMIC_1BYTE_WRITE(&low_on_memory_pending, value, false);
trigger = true;
}
release_recursive_lock(&low_on_memory_pending_lock);
if (trigger)
instrument_low_on_memory();
}
static void
schedule_low_on_memory_event_trigger()
{
bool value = true;
ATOMIC_1BYTE_WRITE(&low_on_memory_pending, value, false);
}
static vm_addr_t
vmm_heap_reserve(size_t size, heap_error_code_t *error_code, bool executable,
which_vmm_t which)
{
vm_addr_t p;
vm_heap_t *vmh = vmheap_for_which(which);
ASSERT(size > 0 && ALIGNED(size, PAGE_SIZE));
ASSERT(!OWN_MUTEX(&reset_pending_lock));
if (DYNAMO_OPTION(vm_reserve)) {
if (INTERNAL_OPTION(vm_use_last) ||
(DYNAMO_OPTION(switch_to_os_at_vmm_reset_limit) &&
at_reset_at_vmm_limit(vmh))) {
DO_ONCE({
if (DYNAMO_OPTION(reset_at_switch_to_os_at_vmm_limit)) {
schedule_reset(RESET_ALL);
}
schedule_low_on_memory_event_trigger();
DOCHECK(1, {
if (!INTERNAL_OPTION(vm_use_last)) {
ASSERT_CURIOSITY(false && "running low on vm reserve");
}
});
* notification of this ... */
});
reached_beyond_vmm(which);
#ifdef X64
if (TEST(VMM_REACHABLE, which) || REACHABLE_HEAP()) {
* contraints */
p = os_heap_reserve_in_region(
(void *)ALIGN_FORWARD(heap_allowable_region_start, PAGE_SIZE),
(void *)ALIGN_BACKWARD(heap_allowable_region_end, PAGE_SIZE), size,
error_code, executable);
if (p != NULL)
request_region_be_heap_reachable(p, size);
} else
p = os_heap_reserve(NULL, size, error_code, executable);
#else
p = os_heap_reserve(NULL, size, error_code, executable);
#endif
if (p != NULL)
return p;
LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_reserve %s: failed " PFX "\n", vmh->name,
*error_code);
}
if (at_reset_at_vmm_limit(vmh)) {
schedule_low_on_memory_event_trigger();
if (schedule_reset(RESET_ALL)) {
STATS_INC(reset_low_vmm_count);
DO_THRESHOLD_SAFE(
DYNAMO_OPTION(report_reset_vmm_threshold), FREQ_PROTECTED_SECTION,
{ },
{
* how much thrashing there is? */
DO_ONCE({
SYSLOG_CUSTOM_NOTIFY(SYSLOG_WARNING, MSG_LOW_ON_VMM_MEMORY, 2,
"Potentially thrashing on low virtual "
"memory resetting.",
get_application_name(),
get_application_pid());
ASSERT_CURIOSITY(false && "vmm heap limit reset thrashing");
});
});
}
}
p = vmm_heap_reserve_blocks(vmh, size, NULL, which);
LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_reserve %s: size=%d p=" PFX "\n", vmh->name,
size, p);
if (p != NULL) {
if (DYNAMO_OPTION(satisfy_w_xor_x) && vmh == &heapmgt->vmcode &&
!executable) {
* Then things like reachable heap do not need to convert to
* writable all over the place.
*/
p = (p - vmh->start_addr) + heapmgt->vmcode_writable_base;
}
return p;
}
DO_ONCE({
DODEBUG({ out_of_vmheap_once = true; });
if (!INTERNAL_OPTION(skip_out_of_vm_reserve_curiosity)) {
SYSLOG_INTERNAL_WARNING("Out of %s reservation - reserving %dKB. "
"Falling back onto OS allocation",
(TEST(VMM_REACHABLE, which) || REACHABLE_HEAP())
? "vmcode"
: "vmheap",
size / 1024);
ASSERT_CURIOSITY(false && "Out of vmheap reservation");
}
* thread-private configuration being a memory hog (and thus we use up
* our reserve). Reset needs memory, and this is asynchronous, so no
* guarantees here anyway (the app may have already reserved all memory
* beyond our reservation, see sqlsrvr.exe and cisvc.exe for ex.) which is
* why we have -reset_at_vmm_threshold to make reset success more likely. */
if (DYNAMO_OPTION(reset_at_vmm_full)) {
schedule_reset(RESET_ALL);
}
});
}
reached_beyond_vmm(which);
#ifdef X64
if (TEST(VMM_REACHABLE, which) || REACHABLE_HEAP()) {
p = os_heap_reserve_in_region(
(void *)ALIGN_FORWARD(heap_allowable_region_start, PAGE_SIZE),
(void *)ALIGN_BACKWARD(heap_allowable_region_end, PAGE_SIZE), size,
error_code, executable);
if (p != NULL)
request_region_be_heap_reachable(p, size);
} else
p = os_heap_reserve(NULL, size, error_code, executable);
#else
p = os_heap_reserve(NULL, size, error_code, executable);
#endif
return p;
}
* This is here just to complement the vmm interface, in fact it is
* almost an alias for os_heap_commit. (If we had strict types then
* here we'd convert a vm_addr_t into a heap_pc.)
*/
static inline bool
vmm_heap_commit(vm_addr_t p, size_t size, uint prot, heap_error_code_t *error_code,
which_vmm_t which)
{
bool res = true;
vm_heap_t *vmh = vmheap_for_which(which);
LOG(GLOBAL, LOG_HEAP, 3, "vmm_heap_commit %s: size=%d p=" PFX " prot=%x\n", vmh->name,
size, p, prot);
if (DYNAMO_OPTION(satisfy_w_xor_x) && vmh == &heapmgt->vmcode) {
vm_addr_t p_writable = vmm_normalize_addr(vmh, &p);
uint shadow_prot = prot & ~(MEMPROT_EXEC);
res = os_heap_commit(p_writable, size, shadow_prot, error_code);
prot &= ~(MEMPROT_WRITE);
if (res) {
* execution from regions allocated executable, not changed to executable.
* There is a downside: IMA policies can cause a significant (~5s) delay
* while a hash is computed of our vmcode region on the first +x mmap.
* Today os_create_memory_file() does a temporary +x mmap for us, avoiding
* any cost here.
*/
size_t map_size = size;
size_t map_offs = p - vmh->start_addr;
vm_addr_t map_addr =
os_map_file(heapmgt->dual_map_file, &map_size, map_offs, p, prot,
MAP_FILE_VMM_COMMIT | MAP_FILE_FIXED);
ASSERT(map_size == size);
res = (map_addr != NULL);
ASSERT(map_addr == NULL || map_addr == p);
}
} else
res = os_heap_commit(p, size, prot, error_code);
size_t commit_used, commit_limit;
ASSERT(!OWN_MUTEX(&reset_pending_lock));
if ((DYNAMO_OPTION(reset_at_commit_percent_free_limit) != 0 ||
DYNAMO_OPTION(reset_at_commit_free_limit) != 0) &&
os_heap_get_commit_limit(&commit_used, &commit_limit)) {
size_t commit_left = commit_limit - commit_used;
ASSERT(commit_used <= commit_limit);
* an issue till 160GB of committable memory. */
if ((DYNAMO_OPTION(reset_at_commit_free_limit) != 0 &&
commit_left < DYNAMO_OPTION(reset_at_commit_free_limit) / PAGE_SIZE) ||
(DYNAMO_OPTION(reset_at_commit_percent_free_limit) != 0 &&
100 * commit_left <
DYNAMO_OPTION(reset_at_commit_percent_free_limit) * commit_limit)) {
* do we want a version of reset that doesn't de-commit our already grabbed
* memory to avoid someone else stealing it (or perhaps keep just a minimal
* level to ensure we make some progress)? */
* telling if we're running in a job and if so what the commit limit for the
* job is. */
* past the threshold we will just spin resetting here and not make any
* progress, may be better to only reset when we have a reasonable amount of
* non-persistent memory to free (so that we can at least make some progress
* before resetting again). */
* it's possible that the pagefile is expandable (dependent on disk space of
* course) and thus we're preventing a potentially beneficial (to us)
* upsizing of the pagefile here. See "HKLM\SYSTEM\CCS\ControlSession /
* Manager\Memory Management" for the initial/max size of the various page
* files (query SystemPafefileInformation only gets you the current size). */
if (schedule_reset(RESET_ALL)) {
STATS_INC(reset_low_commit_count);
DO_THRESHOLD_SAFE(
DYNAMO_OPTION(report_reset_commit_threshold), FREQ_PROTECTED_SECTION,
{ },
{
* how much thrashing there is? */
DO_ONCE({
SYSLOG_CUSTOM_NOTIFY(
SYSLOG_WARNING, MSG_LOW_ON_COMMITTABLE_MEMORY, 2,
"Potentially thrashing on low committable "
"memory resetting.",
get_application_name(), get_application_pid());
ASSERT_CURIOSITY(false && "commit limit reset thrashing");
});
});
}
}
}
if (!res && DYNAMO_OPTION(oom_timeout) != 0 &&
!(DYNAMO_OPTION(satisfy_w_xor_x) && vmh == &heapmgt->vmcode)) {
DEBUG_DECLARE(heap_error_code_t old_error_code = *error_code;)
ASSERT(old_error_code != HEAP_ERROR_SUCCESS);
if (!os_heap_systemwide_overcommit(*error_code)) {
* miniscule chance another thread will free up some or
* will attempt suicide, so could retry even if current
* process has a leak */
ASSERT_NOT_IMPLEMENTED(false);
}
SYSLOG_INTERNAL_WARNING("vmm_heap_commit oom: timeout and retry");
os_timeout(DYNAMO_OPTION(oom_timeout));
res = os_heap_commit(p, size, prot, error_code);
DODEBUG({
if (res) {
SYSLOG_INTERNAL_WARNING("vmm_heap_commit retried, got away! old=" PFX
" new=" PFX "\n",
old_error_code, *error_code);
} else {
SYSLOG_INTERNAL_WARNING("vmm_heap_commit retrying, no luck. old=" PFX
" new=" PFX "\n",
old_error_code, *error_code);
}
});
}
return res;
}
END_DATA_SECTION()
* it is within the memory managed by the virtual memory manager we
* only decommit back to the OS, and we remove the vmm reservation.
* Keep in mind that this can be called on units that are not fully
* committed, e.g. guard pages are added to this - as long as the
* os_heap_decommit interface can handle this we're OK
*/
static void
vmm_heap_free(vm_addr_t p, size_t size, heap_error_code_t *error_code, which_vmm_t which)
{
vm_heap_t *vmh = vmheap_for_which(which);
LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_free %s: size=%d p=" PFX " is_reserved=%d\n",
vmh->name, size, p, vmm_is_reserved_unit(vmh, p, size));
vm_addr_t p_writable = p;
if (DYNAMO_OPTION(satisfy_w_xor_x) && vmh == &heapmgt->vmcode)
p_writable = vmm_normalize_addr(vmh, &p);
* was allocated as an extra OS call when if we ran out.
*/
if (DYNAMO_OPTION(vm_reserve)) {
if (vmm_is_reserved_unit(vmh, p, size)) {
if (DYNAMO_OPTION(satisfy_w_xor_x) && vmh == &heapmgt->vmcode)
os_heap_decommit(p_writable, size, error_code);
os_heap_decommit(p, size, error_code);
vmm_heap_free_blocks(vmh, p, size, which);
LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_free %s: freed size=%d p=" PFX "\n",
vmh->name, size, p);
return;
} else {
after cleanup to a different combination that start_pc = end_pc = NULL
*/
if (vmm_heap_exited) {
*error_code = HEAP_ERROR_SUCCESS;
return;
}
}
}
if (DYNAMO_OPTION(satisfy_w_xor_x) && vmh == &heapmgt->vmcode)
os_heap_free(p_writable, size, error_code);
os_heap_free(p, size, error_code);
}
static void
vmm_heap_decommit(vm_addr_t p, size_t size, heap_error_code_t *error_code,
which_vmm_t which)
{
LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_decommit: size=%d p=" PFX " is_reserved=%d\n",
size, p, is_vmm_reserved_address(p, size, NULL, NULL));
if (DYNAMO_OPTION(satisfy_w_xor_x)) {
vm_heap_t *vmh = vmheap_for_which(which);
if (vmh == &heapmgt->vmcode) {
vm_addr_t p_writable = vmm_normalize_addr(vmh, &p);
os_heap_decommit(p_writable, size, error_code);
}
}
os_heap_decommit(p, size, error_code);
}
* size must be PAGE_SIZE-aligned.
* Returns NULL if fails to allocate memory!
*/
static void *
vmm_heap_alloc(size_t size, uint prot, heap_error_code_t *error_code, which_vmm_t which)
{
vm_addr_t p = vmm_heap_reserve(size, error_code, TEST(MEMPROT_EXEC, prot), which);
if (!p)
return NULL;
if (!vmm_heap_commit(p, size, prot, error_code, which))
return NULL;
return p;
}
void
vmm_heap_init()
{
IF_WINDOWS(ASSERT(ALIGNED(OS_ALLOC_GRANULARITY, DYNAMO_OPTION(vmm_block_size))));
#ifdef X64
* reachable by the vmheap reservation. But, for -heap_in_lower_4GB,
* we must call request_region_be_heap_reachable() up front.
* This is a hard requirement so we set it prior to locating the vmm region.
*/
if (DYNAMO_OPTION(heap_in_lower_4GB))
request_region_be_heap_reachable(0, 0x80000000);
#endif
if (DYNAMO_OPTION(vm_reserve)) {
vmm_heap_unit_init(&heapmgt->vmcode, DYNAMO_OPTION(vm_size), true, "vmcode");
if (!REACHABLE_HEAP()) {
vmm_heap_unit_init(
&heapmgt->vmheap,
IF_WINDOWS_ELSE(IF_X64_ELSE(is_wow64_process(NT_CURRENT_PROCESS)
? DYNAMO_OPTION(vmheap_size_wow64)
: DYNAMO_OPTION(vmheap_size),
DYNAMO_OPTION(vmheap_size)),
DYNAMO_OPTION(vmheap_size)),
false, "vmheap");
}
}
}
static void
vmh_exit(vm_heap_t *vmh, bool contains_stacks)
{
* stack, global_do_syscall.
*/
DOCHECK(1, {
uint perstack =
(uint)(ALIGN_FORWARD_UINT(
DYNAMO_OPTION(stack_size) +
(has_guard_pages(VMM_STACK | VMM_PER_THREAD)
? (2 * PAGE_SIZE)
: (DYNAMO_OPTION(stack_guard_pages) ? PAGE_SIZE : 0)),
DYNAMO_OPTION(vmm_block_size)) /
DYNAMO_OPTION(vmm_block_size));
uint unfreed_blocks;
if (!contains_stacks || standalone_library)
unfreed_blocks = 0;
else {
unfreed_blocks = perstack * 1 +
perstack * ((doing_detach IF_APP_EXPORTS(|| dr_api_exit)) ? 0 : 1);
}
size_t blocks_sz_bytes =
ALIGN_FORWARD_UINT(BITMAP_INDEX(vmh->num_blocks) * sizeof(bitmap_element_t),
DYNAMO_OPTION(vmm_block_size));
unfreed_blocks += (uint)(blocks_sz_bytes / DYNAMO_OPTION(vmm_block_size));
* left behind all TPCs needed so then we can assert even for
* detach.
*/
ASSERT(IF_WINDOWS(doing_detach ||)
vmh->num_free_blocks == vmh->num_blocks - unfreed_blocks ||
* could be outside of vmm (i#1164).
*/
((ever_beyond_vmm
IF_WINDOWS(|| get_os_version() >= WINDOWS_VERSION_8_1)) &&
vmh->num_free_blocks >= vmh->num_blocks - unfreed_blocks));
});
* stack in this region so we cannot free the whole allocation.
* XXX: Any tombstone allocations will have to use a
* different interface than the generic heap_mmap() which is
* sometimes used to leave things behind. FIXME: Currently
* we'll leave behind the whole vm unit if any tombstones are
* left - which in fact is always the case, no matter whether
* thread private code needs to be left or not.
* global_do_syscall 32 byte allocation should be part of our
* dll and won't have to be left.
* The current stack is the main problem because it is later
* cleaned up in cleanup_and_terminate by calling stack_free which
* in turn gets all the way to vmm_heap_free. Therefore we add an
* explicit test for vmm_heap_exited, so that we can otherwise free
* bookkeeping information and delete the lock now.
* Potential solution to most of these problems is to have
* cleanup_and_terminate call vmm_heap_exit when cleaning up
* the process, or to just leave the vm mapping behind and
* simply pass a different argument to stack_free.
*/
vmm_heap_unit_exit(vmh);
}
void
vmm_heap_exit()
{
if (DYNAMO_OPTION(vm_reserve)) {
if (heapmgt->vmcode.start_addr != NULL)
vmh_exit(&heapmgt->vmcode, heapmgt->vmheap.start_addr == NULL);
if (heapmgt->vmheap.start_addr != NULL)
vmh_exit(&heapmgt->vmheap, true);
vmm_heap_exited = true;
}
}
#ifdef UNIX
void
vmm_heap_fork_pre(dcontext_t *dcontext)
{
if (!DYNAMO_OPTION(satisfy_w_xor_x))
return;
* sharing the parent's. Unfortunately that requires copying the entire
* vmcode contents into new mappings. To avoid a race while the child makes
* this copy from our live mappings, we create a temp copy now. The
* disadvantage is that we need a bunch of free memory (and address space:
* but this is 64-bit-only). The alternative is to have the parent wait for
* the child but that seems too disruptive to scheduling.
*/
thread_heap_t *th = (thread_heap_t *)dcontext->heap_field;
heap_error_code_t error_code;
th->fork_copy_size = heapmgt->vmcode.alloc_size;
th->fork_copy_start =
os_heap_reserve(NULL, th->fork_copy_size, &error_code, true );
if (th->fork_copy_start == NULL) {
report_w_xor_x_fatal_error_and_exit();
ASSERT_NOT_REACHED();
}
* are +rw (ELF data segments), some are +rx, and some are +r (reachable
* (non-exec) heap)). We can't use the actual page prot of the copy to store
* what the vmcode prot should be, because some W^X implementations remove +x
* from a +wx region, and we require +w to make our copy. Thus we store the
* mapping prots in a vmvector.
*/
VMVECTOR_ALLOC_VECTOR(th->fork_copy_areas, dcontext,
VECTOR_NEVER_MERGE | VECTOR_NO_LOCK, innermost_lock);
memquery_iter_t iter;
if (!memquery_iterator_start(&iter, heapmgt->vmcode.alloc_start,
true )) {
report_w_xor_x_fatal_error_and_exit();
ASSERT_NOT_REACHED();
}
while (memquery_iterator_next(&iter) && iter.vm_start < heapmgt->vmcode.end_addr) {
if (iter.vm_start < heapmgt->vmcode.alloc_start || iter.prot == MEMPROT_NONE)
continue;
byte *new_start =
iter.vm_start - heapmgt->vmcode.alloc_start + th->fork_copy_start;
vmvector_add(th->fork_copy_areas, new_start,
new_start + (iter.vm_end - iter.vm_start),
(void *)(ptr_uint_t)iter.prot);
if (!os_heap_commit(new_start, iter.vm_end - iter.vm_start,
MEMPROT_READ | MEMPROT_WRITE, &error_code)) {
report_w_xor_x_fatal_error_and_exit();
ASSERT_NOT_REACHED();
}
memcpy(new_start, iter.vm_start, iter.vm_end - iter.vm_start);
LOG(GLOBAL, LOG_HEAP, 2, "%s: copied %p-%p %x to %p-%p\n", __FUNCTION__,
iter.vm_start, iter.vm_end, iter.prot, new_start,
new_start + (iter.vm_end - iter.vm_start));
}
memquery_iterator_stop(&iter);
}
void
vmm_heap_fork_post(dcontext_t *dcontext, bool parent)
{
if (!DYNAMO_OPTION(satisfy_w_xor_x) || !parent)
return;
thread_heap_t *th = (thread_heap_t *)dcontext->heap_field;
heap_error_code_t error_code;
os_heap_free(th->fork_copy_start, th->fork_copy_size, &error_code);
if (error_code != HEAP_ERROR_SUCCESS) {
report_w_xor_x_fatal_error_and_exit();
ASSERT_NOT_REACHED();
}
th->fork_copy_start = NULL;
th->fork_copy_size = 0;
vmvector_reset_vector(dcontext, th->fork_copy_areas);
vmvector_delete_vector(dcontext, th->fork_copy_areas);
th->fork_copy_areas = NULL;
}
void
vmm_heap_fork_init(dcontext_t *dcontext)
{
if (!DYNAMO_OPTION(satisfy_w_xor_x))
return;
* parent's. Unfortunately that requires copying the entire vmcode contents
* into new mappings. The parent has made a temp copy for us to avoid races
* if we tried to copy its live memory.
*/
int old_fd = heapmgt->dual_map_file;
heapmgt->dual_map_file =
os_create_memory_file(MEMORY_FILE_NAME, heapmgt->vmcode.alloc_size);
if (heapmgt->dual_map_file == INVALID_FILE)
goto vmm_heap_fork_init_failed;
LOG(GLOBAL, LOG_HEAP, 2, "%s: new dual_map_file is %d\n", __FUNCTION__,
heapmgt->dual_map_file);
size_t map_size = heapmgt->vmcode.alloc_size;
byte *map_base =
os_map_file(heapmgt->dual_map_file, &map_size, 0, heapmgt->vmcode_writable_alloc,
MEMPROT_NONE, MAP_FILE_VMM_COMMIT | MAP_FILE_FIXED);
if (map_base != heapmgt->vmcode_writable_alloc ||
map_size != heapmgt->vmcode.alloc_size)
goto vmm_heap_fork_init_failed;
heap_error_code_t error_code;
thread_heap_t *th = (thread_heap_t *)dcontext->heap_field;
vmvector_iterator_t vmvi;
vmvector_iterator_start(th->fork_copy_areas, &vmvi);
while (vmvector_iterator_hasnext(&vmvi)) {
byte *start, *end;
uint prot = (uint)(ptr_uint_t)vmvector_iterator_next(&vmvi, &start, &end);
byte *new_start = start - th->fork_copy_start + heapmgt->vmcode_writable_alloc;
uint new_prot = (prot & ~(MEMPROT_EXEC)) | MEMPROT_WRITE;
if (!os_heap_commit(new_start, end - start, new_prot, &error_code))
goto vmm_heap_fork_init_failed;
memcpy(new_start, start, end - start);
LOG(GLOBAL, LOG_HEAP, 2, "%s: re-mapped %p-%p %x; copied from %p-%p %x\n",
__FUNCTION__, new_start, new_start + (end - start), new_prot, start, end,
prot);
}
vmvector_iterator_stop(&vmvi);
map_size = heapmgt->vmcode.alloc_size;
map_base =
os_map_file(heapmgt->dual_map_file, &map_size, 0, heapmgt->vmcode.alloc_start,
MEMPROT_NONE, MAP_FILE_VMM_COMMIT | MAP_FILE_FIXED);
if (map_base != heapmgt->vmcode.alloc_start || map_size != heapmgt->vmcode.alloc_size)
goto vmm_heap_fork_init_failed;
vmvector_iterator_start(th->fork_copy_areas, &vmvi);
while (vmvector_iterator_hasnext(&vmvi)) {
byte *start, *end;
uint prot = (uint)(ptr_uint_t)vmvector_iterator_next(&vmvi, &start, &end);
byte *new_start = start - th->fork_copy_start + heapmgt->vmcode.alloc_start;
map_size = end - start;
map_base =
os_map_file(heapmgt->dual_map_file, &map_size, start - th->fork_copy_start,
new_start, prot, MAP_FILE_VMM_COMMIT | MAP_FILE_FIXED);
if (map_base != new_start || map_size != end - start)
goto vmm_heap_fork_init_failed;
LOG(GLOBAL, LOG_HEAP, 2, "%s: re-mapped %p-%p %x\n", __FUNCTION__, new_start,
new_start + map_size, prot);
}
vmvector_iterator_stop(&vmvi);
os_heap_free(th->fork_copy_start, th->fork_copy_size, &error_code);
if (error_code != HEAP_ERROR_SUCCESS)
goto vmm_heap_fork_init_failed;
th->fork_copy_start = NULL;
th->fork_copy_size = 0;
vmvector_reset_vector(dcontext, th->fork_copy_areas);
vmvector_delete_vector(dcontext, th->fork_copy_areas);
th->fork_copy_areas = NULL;
* os_delete_memory_file(). This may not work on Windows if that function needs to do
* more.
*/
os_close_protected(old_fd);
return;
vmm_heap_fork_init_failed:
report_w_xor_x_fatal_error_and_exit();
ASSERT_NOT_REACHED();
}
#endif
* modified the value of any options to make them compatible
*/
bool
heap_check_option_compatibility()
{
bool ret = false;
ret = check_param_bounds(&dynamo_options.vm_size, MIN_VMM_HEAP_UNIT_SIZE,
MAX_VMCODE_SIZE, "vm_size") ||
ret;
ret = check_param_bounds(&dynamo_options.vmheap_size, MIN_VMM_HEAP_UNIT_SIZE,
MAX_VMHEAP_SIZE, "vmheap_size") ||
ret;
#ifdef INTERNAL
* "initial_heap_unit_size must be >= 8229 and <= 4096" but in
* release build we will take the min and then complain about
* max_heap_unit_size and set it to the min also, so it all works
* out w/o needing an extra check() call.
*/
ret = check_param_bounds(&dynamo_options.initial_heap_unit_size,
* allocating 64KB chunks for "oversized" units
* for just about every alloc! so round up to
* at least a page.
*/
ALIGN_FORWARD(UNITOVERHEAD + 1, (uint)PAGE_SIZE),
HEAP_UNIT_MAX_SIZE, "initial_heap_unit_size") ||
ret;
ret = check_param_bounds(&dynamo_options.initial_global_heap_unit_size,
ALIGN_FORWARD(UNITOVERHEAD + 1, (uint)PAGE_SIZE),
HEAP_UNIT_MAX_SIZE, "initial_global_heap_unit_size") ||
ret;
ret = check_param_bounds(&dynamo_options.max_heap_unit_size,
MAX(HEAP_UNIT_MIN_SIZE, GLOBAL_UNIT_MIN_SIZE), INT_MAX,
"max_heap_unit_size") ||
ret;
#endif
return ret;
}
void
heap_reset_init()
{
threadunits_init(GLOBAL_DCONTEXT, &heapmgt->global_nonpersistent_units,
GLOBAL_UNIT_MIN_SIZE, false);
}
void
d_r_heap_init()
{
int i;
DEBUG_DECLARE(uint prev_sz = 0;)
LOG(GLOBAL, LOG_TOP | LOG_HEAP, 2, "Heap bucket sizes are:\n");
ASSERT(ALIGNED(HEADER_SIZE, HEAP_ALIGNMENT));
ASSERT(BLOCK_SIZES[0] >= sizeof(heap_pc *));
for (i = 0; i < BLOCK_TYPES; i++) {
ASSERT(BLOCK_SIZES[i] > prev_sz);
ASSERT(i == BLOCK_TYPES - 1 || ALIGNED(BLOCK_SIZES[i], HEAP_ALIGNMENT));
DODEBUG(prev_sz = BLOCK_SIZES[i];);
LOG(GLOBAL, LOG_TOP | LOG_HEAP, 2, "\t%d bytes\n", BLOCK_SIZES[i]);
}
* i.e., the vars don't cross cache lines. they shouldn't since
* they should all be 4-byte-aligned in the data segment.
* FIXME: ensure that release build aligns ok?
* I would be quite surprised if static vars were not 4-byte-aligned!
*/
ASSERT(ALIGN_BACKWARD(&heap_exiting, CACHE_LINE_SIZE()) ==
ALIGN_BACKWARD(&heap_exiting + 1, CACHE_LINE_SIZE()));
ASSERT(ALIGN_BACKWARD(&heap_unit_lock.owner, CACHE_LINE_SIZE()) ==
ALIGN_BACKWARD(&heap_unit_lock.owner + 1, CACHE_LINE_SIZE()));
* our post-init heapmgt struct
*/
ASSERT(heapmgt == &temp_heapmgt);
heapmgt->global_heap_writable = true;
threadunits_init(GLOBAL_DCONTEXT, &heapmgt->global_units, GLOBAL_UNIT_MIN_SIZE,
false);
heapmgt =
HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, heap_management_t, ACCT_MEM_MGT, PROTECTED);
ASSERT(sizeof(temp_heapmgt) == sizeof(*heapmgt));
memcpy(heapmgt, &temp_heapmgt, sizeof(temp_heapmgt));
threadunits_init(GLOBAL_DCONTEXT, &heapmgt->global_unprotected_units,
GLOBAL_UNIT_MIN_SIZE, false);
if (!REACHABLE_HEAP()) {
threadunits_init(GLOBAL_DCONTEXT, &heapmgt->global_reachable_units,
GLOBAL_UNIT_MIN_SIZE, true);
}
heap_reset_init();
#ifdef WINDOWS
* using landing pads (see win32/callback.c for more explanation). Landing
* pad areas are a type of special heap, so they should be initialized
* during heap init.
* Each landing pad area has its own allocation pointer, so they shouldn't
* be merged automatically.
*/
VMVECTOR_ALLOC_VECTOR(landing_pad_areas, GLOBAL_DCONTEXT,
VECTOR_SHARED | VECTOR_NEVER_MERGE, landing_pad_areas_lock);
#endif
}
static void
really_free_unit(heap_unit_t *u)
{
RSTATS_SUB(heap_capacity, UNIT_COMMIT_SIZE(u));
STATS_ADD(heap_reserved_only,
(stats_int_t)(UNIT_COMMIT_SIZE(u) - UNIT_RESERVED_SIZE(u)));
release_guarded_real_memory((vm_addr_t)u, UNIT_RESERVED_SIZE(u),
false , true, u->which);
}
* heap_reset_init() will be called before continuing.
*/
void
heap_reset_free()
{
heap_unit_t *u, *next_u;
ASSERT(DYNAMO_OPTION(enable_reset));
* order violations when freeing
*/
dynamo_vm_areas_lock();
* FIXME: remove if we go to separate stats sum location
*/
DODEBUG({ acquire_recursive_lock(&global_alloc_lock); });
acquire_recursive_lock(&heap_unit_lock);
LOG(GLOBAL, LOG_HEAP, 1, "Pre-reset, global heap unit stats:\n");
threadunits_exit(&heapmgt->global_nonpersistent_units, GLOBAL_DCONTEXT);
u = heapmgt->heap.dead;
while (u != NULL) {
next_u = u->next_global;
LOG(GLOBAL, LOG_HEAP, 1, "\tfreeing dead unit " PFX "-" PFX " [-" PFX "]\n", u,
UNIT_COMMIT_END(u), UNIT_RESERVED_END(u));
RSTATS_DEC(heap_num_free);
really_free_unit(u);
u = next_u;
}
heapmgt->heap.dead = NULL;
heapmgt->heap.num_dead = 0;
release_recursive_lock(&heap_unit_lock);
DODEBUG({ release_recursive_lock(&global_alloc_lock); });
dynamo_vm_areas_unlock();
}
void
d_r_heap_exit()
{
heap_unit_t *u, *next_u;
heap_management_t *temp;
heap_exiting = true;
dynamo_vm_areas_lock();
acquire_recursive_lock(&heap_unit_lock);
#ifdef WINDOWS
release_landing_pad_mem();
#endif
LOG(GLOBAL, LOG_HEAP, 1, "Global unprotected heap unit stats:\n");
threadunits_exit(&heapmgt->global_unprotected_units, GLOBAL_DCONTEXT);
LOG(GLOBAL, LOG_HEAP, 1, "Global nonpersistent heap unit stats:\n");
threadunits_exit(&heapmgt->global_nonpersistent_units, GLOBAL_DCONTEXT);
if (!REACHABLE_HEAP()) {
LOG(GLOBAL, LOG_HEAP, 1, "Global reachable heap unit stats:\n");
threadunits_exit(&heapmgt->global_reachable_units, GLOBAL_DCONTEXT);
}
ASSERT(heapmgt != &temp_heapmgt);
* lock structure.
*/
mutex_t temp_vmcode = temp_heapmgt.vmcode.lock;
mutex_t temp_vmheap = temp_heapmgt.vmheap.lock;
memcpy(&temp_heapmgt, heapmgt, sizeof(temp_heapmgt));
temp_heapmgt.vmcode.lock = temp_vmcode;
temp_heapmgt.vmheap.lock = temp_vmheap;
temp = heapmgt;
heapmgt = &temp_heapmgt;
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, temp, heap_management_t, ACCT_MEM_MGT, PROTECTED);
LOG(GLOBAL, LOG_HEAP, 1, "Global heap unit stats:\n");
threadunits_exit(&heapmgt->global_units, GLOBAL_DCONTEXT);
LOG(GLOBAL, LOG_HEAP, 1, "Unfreed units:\n");
u = heapmgt->heap.units;
while (u != NULL) {
next_u = u->next_global;
LOG(GLOBAL, LOG_HEAP, 1, "\tfreeing live unit " PFX "-" PFX " [-" PFX "]\n", u,
UNIT_COMMIT_END(u), UNIT_RESERVED_END(u));
RSTATS_DEC(heap_num_live);
really_free_unit(u);
u = next_u;
}
heapmgt->heap.units = NULL;
u = heapmgt->heap.dead;
while (u != NULL) {
next_u = u->next_global;
LOG(GLOBAL, LOG_HEAP, 1, "\tfreeing dead unit " PFX "-" PFX " [-" PFX "]\n", u,
UNIT_COMMIT_END(u), UNIT_RESERVED_END(u));
RSTATS_DEC(heap_num_free);
really_free_unit(u);
u = next_u;
}
heapmgt->heap.dead = NULL;
heapmgt->global_heap_writable = false;
release_recursive_lock(&heap_unit_lock);
dynamo_vm_areas_unlock();
DELETE_RECURSIVE_LOCK(heap_unit_lock);
DELETE_RECURSIVE_LOCK(global_alloc_lock);
DELETE_RECURSIVE_LOCK(low_on_memory_pending_lock);
#ifdef X64
DELETE_LOCK(request_region_be_heap_reachable_lock);
#endif
if (doing_detach) {
heapmgt = &temp_heapmgt;
IF_X64(reset_heap_reachable_bounds());
}
}
void
heap_post_exit()
{
heap_exiting = false;
}
* detect if the app is who we're fighting for memory, if so, don't
* free memory, else the app will just keep grabbing more.
* need a test for hitting 2GB (or 3GB!) user mode limit.
*/
static void
heap_low_on_memory()
{
heap_unit_t *u, *next_u;
DEBUG_DECLARE(size_t freed = 0;)
LOG(GLOBAL, LOG_CACHE | LOG_STATS, 1,
"heap_low_on_memory: about to free dead list units\n");
* so we have to be careful what locks we grab
* However, no allocation site can hold a lock weaker in rank than
* heap_unit_lock, b/c it could deadlock on the allocation itself!
* So we're safe.
*/
* this is recursive so ok if we ran out of memory while holding DR vm area lock
*/
ASSERT(safe_to_allocate_or_free_heap_units());
dynamo_vm_areas_lock();
acquire_recursive_lock(&heap_unit_lock);
u = heapmgt->heap.dead;
while (u != NULL) {
next_u = u->next_global;
DODEBUG(freed += UNIT_COMMIT_SIZE(u););
LOG(GLOBAL, LOG_HEAP, 1, "\tfreeing dead unit " PFX "-" PFX " [-" PFX "]\n", u,
UNIT_COMMIT_END(u), UNIT_RESERVED_END(u));
RSTATS_DEC(heap_num_free);
really_free_unit(u);
u = next_u;
heapmgt->heap.num_dead--;
}
heapmgt->heap.dead = NULL;
release_recursive_lock(&heap_unit_lock);
dynamo_vm_areas_unlock();
LOG(GLOBAL, LOG_CACHE | LOG_STATS, 1, "heap_low_on_memory: freed %d KB\n",
freed / 1024);
* out or compact at this time.
*/
* of sizes larger than asked for, we may have plenty of memory available
* in other lists! see comments in common_heap_alloc
*/
}
static const char *
get_oom_source_name(oom_source_t source)
{
* (still as a string though)
*/
const char *code_name = "?";
switch (source) {
case OOM_INIT: code_name = "I"; break;
case OOM_RESERVE: code_name = "R"; break;
case OOM_COMMIT: code_name = "C"; break;
case OOM_EXTEND: code_name = "E"; break;
default: ASSERT_NOT_REACHED();
}
return code_name;
}
static bool
silent_oom_for_process(oom_source_t source)
{
if (TESTANY(OOM_COMMIT | OOM_EXTEND, source) &&
!IS_STRING_OPTION_EMPTY(silent_commit_oom_list)) {
bool onlist;
const char *process_name = get_short_name(get_application_name());
string_option_read_lock();
onlist = check_filter_with_wildcards(DYNAMO_OPTION(silent_commit_oom_list),
process_name);
string_option_read_unlock();
if (onlist) {
SYSLOG_INTERNAL_WARNING("not reporting last words of executable %s",
process_name);
return true;
}
}
return false;
}
* the returned value from the last system call - opaque at this OS
* independent layer.
*/
static void
report_low_on_memory(which_vmm_t which, oom_source_t source,
heap_error_code_t os_error_code)
{
if (TESTANY(DYNAMO_OPTION(silent_oom_mask), source) ||
silent_oom_for_process(source)) {
SYSLOG_INTERNAL_WARNING("Mostly silent OOM: %s " PFX ".\n",
get_oom_source_name(source), os_error_code);
if (TEST(DUMPCORE_OUT_OF_MEM_SILENT, DYNAMO_OPTION(dumpcore_mask)))
os_dump_core("Out of memory, silently aborting program.");
} else {
const char *oom_source_code = get_oom_source_name(source);
char type_hex[19];
snprintf(type_hex, BUFFER_SIZE_ELEMENTS(type_hex), PFX, which);
NULL_TERMINATE_BUFFER(type_hex);
char status_hex[19];
snprintf(status_hex, BUFFER_SIZE_ELEMENTS(status_hex), PFX, os_error_code);
NULL_TERMINATE_BUFFER(status_hex);
SYSLOG(SYSLOG_CRITICAL, OUT_OF_MEMORY, 4, get_application_name(),
get_application_pid(), oom_source_code, type_hex, status_hex);
if (INTERNAL_OPTION(rstats_to_stderr))
dump_global_rstats_to_stderr();
if (TEST(DUMPCORE_OUT_OF_MEM, DYNAMO_OPTION(dumpcore_mask)))
os_dump_core("Out of memory, aborting program.");
report_diagnostics("Out of memory", status_hex, NO_VIOLATION_BAD_INTERNAL_STATE);
}
os_terminate(NULL, TERMINATE_PROCESS);
ASSERT_NOT_REACHED();
}
static inline void
account_for_memory(void *p, size_t size, uint prot, bool add_vm,
bool image _IF_DEBUG(const char *comment))
{
RSTATS_ADD_PEAK(memory_capacity, size);
* for clients that use DR-allocated memory, we have get_memory_info()
* query from the OS to see inside
*/
if (is_vmm_reserved_address(p, size, NULL, NULL)) {
return;
}
if (add_vm) {
add_dynamo_vm_area(p, ((app_pc)p) + size, prot, image _IF_DEBUG(comment));
} else {
* add_dynamo_vm_area here, instead we indicate that something has changed
*/
mark_dynamo_vm_areas_stale();
* heap_vmareas_synch_units to update all_memory_areas. Currently
* heap_create_unit is the only place that passes 'false' with prot rw-.
*/
ASSERT(TESTALL(MEMPROT_READ | MEMPROT_WRITE, prot));
}
}
static void
update_dynamo_areas_on_release(app_pc start, app_pc end, bool remove_vm)
{
if (!vm_areas_exited && !heap_exiting) {
* for clients that use DR-allocated memory, we have get_memory_info()
* query from the OS to see inside
*/
if (is_vmm_reserved_address(start, end - start, NULL, NULL)) {
return;
}
if (remove_vm) {
remove_dynamo_vm_area(start, end);
} else {
* incrementally. The pending set is protected by the same lock
* needed to synch the vm areas, so we will never mis-identify free
* memory as DR memory.
*/
mark_dynamo_vm_areas_stale();
dynamo_areas_pending_remove = true;
}
}
}
bool
lockwise_safe_to_allocate_memory()
{
* for memory allocation -- i.e., check whether we hold the
* global_alloc_lock
*/
return !self_owns_recursive_lock(&global_alloc_lock);
}
* mapping a file. First attempts to reserve at "preferred" but if that fails it
* attempts at any available location.
*/
byte *
heap_reserve_for_external_mapping(byte *preferred, size_t size, which_vmm_t which)
{
#ifdef WINDOWS
* on top of an existing reservation; nor can we un-reserve a piece of a
* reservation. See the issue for solution ideas.
*/
ASSERT_NOT_IMPLEMENTED(false && "i#3570");
return NULL;
#endif
vm_addr_t p = NULL;
vm_heap_t *vmh = vmheap_for_which(which);
ASSERT(size > 0);
size = ALIGN_FORWARD(size, PAGE_SIZE);
if (!DYNAMO_OPTION(vm_reserve))
return NULL;
if (preferred >= vmh->start_addr && preferred + size <= vmh->end_addr)
p = vmm_heap_reserve_blocks(vmh, size, preferred, which);
if (p == NULL)
p = vmm_heap_reserve_blocks(vmh, size, NULL, which);
LOG(GLOBAL, LOG_HEAP, 2, "%s %s: size=%d p=" PFX "\n", __FUNCTION__, vmh->name, size,
p);
return p;
}
* its state from before heap_reserve_for_external_mapping() was
* called: reserved but not committed.
*/
bool
heap_unreserve_for_external_mapping(byte *p, size_t size, which_vmm_t which)
{
#ifdef WINDOWS
* on top of an existing reservation; nor can we un-reserve a piece of a
* reservation. See the issue for solution ideas.
*/
ASSERT_NOT_IMPLEMENTED(false && "i#3570");
return false;
#endif
vm_heap_t *vmh = vmheap_for_which(which);
ASSERT(size > 0);
size = ALIGN_FORWARD(size, PAGE_SIZE);
if (!DYNAMO_OPTION(vm_reserve) || !is_vmm_reserved_address(p, size, NULL, NULL))
return false;
vmm_heap_free_blocks(vmh, p, size, which);
LOG(GLOBAL, LOG_HEAP, 2, "%s %s: size=%d p=" PFX "\n", __FUNCTION__, vmh->name, size,
p);
return true;
}
* to handle the out-of-memory condition.
* add_vm MUST be false iff this is heap memory, which is updated separately.
*/
static void *
get_real_memory(size_t size, uint prot, bool add_vm,
which_vmm_t which _IF_DEBUG(const char *comment))
{
void *p;
heap_error_code_t error_code;
size = ALIGN_FORWARD(size, PAGE_SIZE);
dynamo_vm_areas_lock();
p = vmm_heap_alloc(size, prot, &error_code, which);
if (p == NULL) {
SYSLOG_INTERNAL_WARNING_ONCE("Out of memory -- cannot reserve or "
"commit %dKB. Trying to recover.",
size / 1024);
* or heap_unit_lock w/o first having grabbed DR areas lock
*/
ASSERT(safe_to_allocate_or_free_heap_units());
heap_low_on_memory();
fcache_low_on_memory();
* FIXME: have more sophisticated strategy of freeing a little, then getting
* more drastic with each subsequent failure
* FIXME: can only free live fcache units for current thread w/ current
* impl...should we wait a while and try again if out of memory, hoping
* other threads have freed some?!?!
*/
p = vmm_heap_alloc(size, prot, &error_code, which);
if (p == NULL) {
report_low_on_memory(which, OOM_RESERVE, error_code);
}
SYSLOG_INTERNAL_WARNING_ONCE("Out of memory -- but still alive after "
"emergency free.");
}
account_for_memory(p, size, prot, add_vm, false _IF_DEBUG(comment));
dynamo_vm_areas_unlock();
return p;
}
static void
release_memory_and_update_areas(app_pc p, size_t size, bool decommit, bool remove_vm,
which_vmm_t which)
{
heap_error_code_t error_code;
dynamo_vm_areas_lock();
* we end up allocating memory in the process of removing the area
* (we don't want to end up getting the memory we just freed since that
* would lead to errors in the list when we finally did remove it)
*/
update_dynamo_areas_on_release(p, p + size, remove_vm);
if (decommit)
vmm_heap_decommit(p, size, &error_code, which);
else
vmm_heap_free(p, size, &error_code, which);
ASSERT(error_code == HEAP_ERROR_SUCCESS);
dynamo_vm_areas_unlock();
}
static void
release_real_memory(void *p, size_t size, bool remove_vm, which_vmm_t which)
{
size = ALIGN_FORWARD(size, PAGE_SIZE);
release_memory_and_update_areas((app_pc)p, size, false , remove_vm, which);
if (IF_DEBUG_ELSE(!dynamo_exited_log_and_stats, true))
RSTATS_SUB(memory_capacity, size);
}
static void
extend_commitment(vm_addr_t p, size_t size, uint prot, bool initial_commit,
which_vmm_t which)
{
heap_error_code_t error_code;
ASSERT(ALIGNED(p, PAGE_SIZE));
size = ALIGN_FORWARD(size, PAGE_SIZE);
if (!vmm_heap_commit(p, size, prot, &error_code, which)) {
SYSLOG_INTERNAL_WARNING_ONCE("Out of memory - cannot extend commit "
"%dKB. Trying to recover.",
size / 1024);
heap_low_on_memory();
fcache_low_on_memory();
if (!vmm_heap_commit(p, size, prot, &error_code, which)) {
report_low_on_memory(which, initial_commit ? OOM_COMMIT : OOM_EXTEND,
error_code);
}
SYSLOG_INTERNAL_WARNING_ONCE("Out of memory in extend - still alive "
"after emergency free.");
}
}
* requested unit. These should consume only uncommitted virtual address and
* should not use any physical memory.
* add_vm MUST be false iff this is heap memory, which is updated separately.
* Non-NULL min_addr is only supported for stack allocations (DrMi#1723).
*/
static vm_addr_t
get_guarded_real_memory(size_t reserve_size, size_t commit_size, uint prot, bool add_vm,
bool guarded, byte *min_addr,
which_vmm_t which _IF_DEBUG(const char *comment))
{
vm_addr_t p = NULL;
uint guard_size = (uint)PAGE_SIZE;
heap_error_code_t error_code;
bool try_vmm = true;
ASSERT(reserve_size >= commit_size);
if (!guarded || !has_guard_pages(which)) {
if (reserve_size == commit_size)
return get_real_memory(reserve_size, prot, add_vm, which _IF_DEBUG(comment));
guard_size = 0;
}
reserve_size = ALIGN_FORWARD(reserve_size, PAGE_SIZE);
commit_size = ALIGN_FORWARD(commit_size, PAGE_SIZE);
reserve_size += 2 * guard_size;
dynamo_vm_areas_lock();
#ifdef WINDOWS
* can trigger an app guard
* page. We have to ensure that the kernel will update TEB.StackLimit in that
* case, which requires our dstack to be higher than the app stack.
* This results in more fragmentation and larger dynamo_areas so we avoid
* if we can. We could consider a 2nd vm_reserve region just for stacks.
*/
if (SWAP_TEB_STACKBASE() && (!DYNAMO_OPTION(vm_reserve) && min_addr > NULL) ||
(DYNAMO_OPTION(vm_reserve) && min_addr > vmheap_get_start())) {
try_vmm = false;
}
#endif
if (try_vmm)
p = vmm_heap_reserve(reserve_size, &error_code, TEST(MEMPROT_EXEC, prot), which);
#ifdef WINDOWS
if (!try_vmm || p < (vm_addr_t)min_addr) {
if (p != NULL)
vmm_heap_free(p, reserve_size, &error_code, which);
p = os_heap_reserve_in_region((void *)ALIGN_FORWARD(min_addr, PAGE_SIZE),
(void *)PAGE_START(POINTER_MAX), reserve_size,
&error_code, TEST(MEMPROT_EXEC, prot));
* (min_addr != NULL assumed to be stack).
*/
ASSERT(!DYNAMO_OPTION(stack_shares_gencode));
* and rely on our other best-effort TEB.StackLimit updating checks
* (check_app_stack_limit()).
*/
if (p == NULL) {
SYSLOG_INTERNAL_WARNING_ONCE("Unable to allocate dstack above app stack");
if (!try_vmm) {
p = vmm_heap_reserve(reserve_size, &error_code, TEST(MEMPROT_EXEC, prot),
which);
}
}
}
#endif
if (p == NULL) {
SYSLOG_INTERNAL_WARNING_ONCE("Out of memory - cannot reserve %dKB. "
"Trying to recover.",
reserve_size / 1024);
heap_low_on_memory();
fcache_low_on_memory();
p = vmm_heap_reserve(reserve_size, &error_code, TEST(MEMPROT_EXEC, prot), which);
if (p == NULL) {
report_low_on_memory(which, OOM_RESERVE, error_code);
}
SYSLOG_INTERNAL_WARNING_ONCE("Out of memory on reserve - but still "
"alive after emergency free.");
}
* add guard pages in by assuming one page on each side of every heap unit
* if dynamo_options.guard_pages
*/
account_for_memory((void *)p, reserve_size, prot, add_vm, false _IF_DEBUG(comment));
dynamo_vm_areas_unlock();
STATS_ADD_PEAK(reserved_memory_capacity, reserve_size);
STATS_ADD_PEAK(guard_pages, 2);
p += guard_size;
extend_commitment(p, commit_size, prot, true , which);
return p;
}
* side of the requested unit. remove_vm MUST be false iff this is heap memory,
* which is updated separately.
*/
static void
release_guarded_real_memory(vm_addr_t p, size_t size, bool remove_vm, bool guarded,
which_vmm_t which)
{
if (!guarded || !has_guard_pages(which)) {
release_real_memory(p, size, remove_vm, which);
return;
}
size = ALIGN_FORWARD(size, PAGE_SIZE);
size += PAGE_SIZE * 2;
p -= PAGE_SIZE;
release_memory_and_update_areas((app_pc)p, size, false , remove_vm, which);
if (IF_DEBUG_ELSE(!dynamo_exited_log_and_stats, true)) {
RSTATS_SUB(memory_capacity, size);
STATS_SUB(reserved_memory_capacity, size);
STATS_ADD(guard_pages, -2);
}
}
* it's mainly used to allocate our fcache units
*/
void *
heap_mmap_ex(size_t reserve_size, size_t commit_size, uint prot, bool guarded,
which_vmm_t which)
{
void *p = get_guarded_real_memory(reserve_size, commit_size, prot, true, guarded,
NULL, which _IF_DEBUG("heap_mmap"));
#ifdef DEBUG_MEMORY
if (TEST(MEMPROT_WRITE, prot))
memset(vmm_get_writable_addr(p, which), HEAP_ALLOCATED_BYTE, commit_size);
#endif
ASSERT(!DYNAMO_OPTION(vm_reserve) || !DYNAMO_OPTION(stack_shares_gencode) ||
(ptr_uint_t)p - (guarded ? (GUARD_PAGE_ADJUSTMENT / 2) : 0) ==
ALIGN_BACKWARD(p, DYNAMO_OPTION(vmm_block_size)) ||
at_reset_at_vmm_limit(vmheap_for_which(which)));
LOG(GLOBAL, LOG_HEAP, 2, "heap_mmap: %d bytes [/ %d] @ " PFX "\n", commit_size,
reserve_size, p);
STATS_ADD_PEAK(mmap_capacity, commit_size);
STATS_ADD_PEAK(mmap_reserved_only, (reserve_size - commit_size));
return p;
}
void *
heap_mmap_reserve(size_t reserve_size, size_t commit_size, uint prot, which_vmm_t which)
{
return heap_mmap_ex(reserve_size, commit_size, prot, true, which);
}
* and that it does not extend beyond the initial reservation.
*/
void
heap_mmap_extend_commitment(void *p, size_t commit_size, which_vmm_t which)
{
extend_commitment(p, commit_size, MEMPROT_EXEC | MEMPROT_READ | MEMPROT_WRITE,
false , which);
STATS_SUB(mmap_reserved_only, commit_size);
STATS_ADD_PEAK(mmap_capacity, commit_size);
#ifdef DEBUG_MEMORY
memset(vmm_get_writable_addr(p, which), HEAP_ALLOCATED_BYTE, commit_size);
#endif
}
void
heap_mmap_retract_commitment(void *retract_start, size_t decommit_size, which_vmm_t which)
{
heap_error_code_t error_code;
ASSERT(ALIGNED(decommit_size, PAGE_SIZE));
vmm_heap_decommit(retract_start, decommit_size, &error_code, which);
STATS_ADD(mmap_reserved_only, decommit_size);
STATS_ADD_PEAK(mmap_capacity, -(stats_int_t)decommit_size);
}
* stack, to save address space (case 9474).
* Doing so is only supported for -reachable_heap (but for x64 we don't have
* private gencode in any case).
*/
void *
heap_mmap_reserve_post_stack(dcontext_t *dcontext, size_t reserve_size,
size_t commit_size, uint prot, which_vmm_t which)
{
void *p;
byte *stack_reserve_end = NULL;
heap_error_code_t error_code;
size_t available = 0;
uint cur_prot = 0;
bool known_stack = false;
vm_heap_t *vmh = vmheap_for_which(which);
ASSERT(reserve_size > 0 && commit_size < reserve_size);
if (DYNAMO_OPTION(stack_size) + reserve_size + GUARD_PAGE_ADJUSTMENT +
GUARD_PAGE_ADJUSTMENT / 2 >
DYNAMO_OPTION(vmm_block_size)) {
LOG(GLOBAL, LOG_HEAP, 1,
"Not enough room to allocate 0x%08x bytes post stack "
"of size 0x%08x\n",
reserve_size, DYNAMO_OPTION(stack_size));
return heap_mmap_reserve(reserve_size, commit_size, prot, which);
}
if (DYNAMO_OPTION(stack_shares_gencode) &&
* the reservation, but we don't bother */
DYNAMO_OPTION(vm_reserve) && dcontext != GLOBAL_DCONTEXT && dcontext != NULL) {
stack_reserve_end = dcontext->dstack + GUARD_PAGE_ADJUSTMENT / 2;
#if defined(UNIX) && !defined(HAVE_MEMINFO)
if (!dynamo_initialized) {
* post-stack if inside vmm (won't be true only for pathological
* tiny vmm sizes)
*/
if (vmm_is_reserved_unit(vmh, stack_reserve_end, reserve_size)) {
known_stack = true;
available = reserve_size;
} else
known_stack = false;
} else
#elif defined(UNIX)
known_stack =
get_memory_info_from_os(stack_reserve_end, NULL, &available, &cur_prot);
#else
known_stack = get_memory_info(stack_reserve_end, NULL, &available, &cur_prot);
#endif
* which we could support but don't (see FIXME above) */
ASSERT(out_of_vmheap_once ||
(known_stack && available >= reserve_size && cur_prot == 0));
}
if (!known_stack ||
!vmm_in_same_block(vmh, dcontext->dstack,
stack_reserve_end + reserve_size) ||
available < reserve_size) {
ASSERT(!DYNAMO_OPTION(stack_shares_gencode) || !DYNAMO_OPTION(vm_reserve) ||
out_of_vmheap_once);
DOLOG(1, LOG_HEAP, {
if (known_stack && available < reserve_size) {
LOG(GLOBAL, LOG_HEAP, 1,
"heap_mmap_reserve_post_stack: avail %d < needed %d\n", available,
reserve_size);
}
});
STATS_INC(mmap_no_share_stack_region);
return heap_mmap_reserve(reserve_size, commit_size, prot, which);
}
ASSERT(DYNAMO_OPTION(vm_reserve));
ASSERT(REACHABLE_HEAP());
ASSERT(stack_reserve_end != NULL);
dynamo_vm_areas_lock();
if (vmm_is_reserved_unit(vmh, stack_reserve_end, reserve_size)) {
p = stack_reserve_end;
} else {
p = os_heap_reserve(stack_reserve_end, reserve_size, &error_code, true );
#ifdef X64
* (this will also verify that this region meets reachability requirements) */
if (p != NULL)
request_region_be_heap_reachable(p, reserve_size);
#endif
if (p == NULL) {
ASSERT_NOT_REACHED();
LOG(GLOBAL, LOG_HEAP, 1,
"heap_mmap_reserve_post_stack: reserve failed " PFX "\n", error_code);
dynamo_vm_areas_unlock();
STATS_INC(mmap_no_share_stack_region);
return heap_mmap_reserve(reserve_size, commit_size, prot, which);
}
ASSERT(error_code == HEAP_ERROR_SUCCESS);
}
if (!vmm_heap_commit(p, commit_size, prot, &error_code, which)) {
ASSERT_NOT_REACHED();
LOG(GLOBAL, LOG_HEAP, 1, "heap_mmap_reserve_post_stack: commit failed " PFX "\n",
error_code);
if (!vmm_is_reserved_unit(vmh, stack_reserve_end, reserve_size)) {
os_heap_free(p, reserve_size, &error_code);
ASSERT(error_code == HEAP_ERROR_SUCCESS);
}
dynamo_vm_areas_unlock();
STATS_INC(mmap_no_share_stack_region);
return heap_mmap_reserve(reserve_size, commit_size, prot, which);
}
account_for_memory(p, reserve_size, prot, true ,
false _IF_DEBUG("heap_mmap_reserve_post_stack"));
dynamo_vm_areas_unlock();
ASSERT((ptr_uint_t)p - GUARD_PAGE_ADJUSTMENT / 2 !=
ALIGN_BACKWARD(p, DYNAMO_OPTION(vmm_block_size)));
#ifdef DEBUG_MEMORY
memset(vmm_get_writable_addr(p, which), HEAP_ALLOCATED_BYTE, commit_size);
#endif
LOG(GLOBAL, LOG_HEAP, 2, "heap_mmap w/ stack: %d bytes [/ %d] @ " PFX "\n",
commit_size, reserve_size, p);
STATS_ADD_PEAK(mmap_capacity, commit_size);
STATS_ADD_PEAK(mmap_reserved_only, (reserve_size - commit_size));
STATS_INC(mmap_share_stack_region);
return p;
}
* thread's stack (case 9474).
*/
void
heap_munmap_post_stack(dcontext_t *dcontext, void *p, size_t reserve_size,
which_vmm_t which)
{
* but on detach we have no dcontext, so we instead use block alignment.
*/
DOCHECK(1, {
if (dcontext != NULL && dcontext != GLOBAL_DCONTEXT &&
DYNAMO_OPTION(vm_reserve) && DYNAMO_OPTION(stack_shares_gencode)) {
bool at_stack_end = (p == dcontext->dstack + GUARD_PAGE_ADJUSTMENT / 2);
bool at_block_start = ((ptr_uint_t)p - GUARD_PAGE_ADJUSTMENT / 2 ==
ALIGN_BACKWARD(p, DYNAMO_OPTION(vmm_block_size)));
ASSERT((at_stack_end && !at_block_start) ||
(!at_stack_end && at_block_start));
}
});
if (!DYNAMO_OPTION(vm_reserve) || !DYNAMO_OPTION(stack_shares_gencode) ||
(ptr_uint_t)p - GUARD_PAGE_ADJUSTMENT / 2 ==
ALIGN_BACKWARD(p, DYNAMO_OPTION(vmm_block_size))) {
heap_munmap(p, reserve_size, which);
} else {
* we use the reserve size, which works fine.
*/
release_memory_and_update_areas((app_pc)p, reserve_size, true ,
true , which);
LOG(GLOBAL, LOG_HEAP, 2, "heap_munmap_post_stack: %d bytes @ " PFX "\n",
reserve_size, p);
STATS_SUB(mmap_capacity, reserve_size);
STATS_SUB(mmap_reserved_only, reserve_size);
}
}
void *
heap_mmap(size_t size, uint prot, which_vmm_t which)
{
return heap_mmap_ex(size, size, prot, true, which);
}
void
heap_munmap_ex(void *p, size_t size, bool guarded, which_vmm_t which)
{
#ifdef DEBUG_MEMORY
* space anymore */
#endif
release_guarded_real_memory((vm_addr_t)p, size, true ,
guarded, which);
DOSTATS({
* dynamo_process_exit
*/
if (!dynamo_exited_log_and_stats) {
LOG(GLOBAL, LOG_HEAP, 2, "heap_munmap: %d bytes @ " PFX "\n", size, p);
STATS_SUB(mmap_capacity, size);
STATS_SUB(mmap_reserved_only, size);
}
});
}
void
heap_munmap(void *p, size_t size, which_vmm_t which)
{
heap_munmap_ex(p, size, true , which);
}
* For -stack_guard_pages, also allocates an extra page
* on the bottom and uses it to detect overflows when accessed.
*/
void *
stack_alloc(size_t size, byte *min_addr)
{
void *p;
* FIXME case 2330: commit-on-demand could allow larger max sizes w/o
* hurting us in the common case
*/
size_t alloc_size = size;
if (!has_guard_pages(VMM_STACK | VMM_PER_THREAD) && DYNAMO_OPTION(stack_guard_pages))
alloc_size += PAGE_SIZE;
p = get_guarded_real_memory(alloc_size, alloc_size, MEMPROT_READ | MEMPROT_WRITE,
true, true, min_addr,
VMM_STACK | VMM_PER_THREAD _IF_DEBUG("stack_alloc"));
if (!has_guard_pages(VMM_STACK | VMM_PER_THREAD) && DYNAMO_OPTION(stack_guard_pages))
p = (byte *)p + PAGE_SIZE;
#ifdef DEBUG_MEMORY
memset(p, HEAP_ALLOCATED_BYTE, size);
#endif
if (DYNAMO_OPTION(stack_guard_pages)) {
* overflow that uses a large stride and skips over one page (UNIX-only
* since Windows code always uses chkstk to trigger guard pages).
*/
* such overflows, we'd rather have a clear crash than memory corruption
* from clobbering whatever memory is below the stack.
*/
byte *guard = (byte *)p - PAGE_SIZE;
#ifdef WINDOWS
* auto-expand the stack into adjacent allocations below the stack.
*/
heap_error_code_t error_code;
if (vmm_heap_commit(guard, PAGE_SIZE, MEMPROT_READ | MEMPROT_WRITE, &error_code,
VMM_STACK | VMM_PER_THREAD))
mark_page_as_guard(guard);
#else
if (!has_guard_pages(VMM_STACK | VMM_PER_THREAD))
set_protection(guard, PAGE_SIZE, MEMPROT_READ);
#endif
}
RSTATS_ADD_PEAK(stack_capacity, size);
return (void *)((ptr_uint_t)p + size);
}
void
stack_free(void *p, size_t size)
{
size_t alloc_size;
if (size == 0)
size = DYNAMORIO_STACK_SIZE;
alloc_size = size;
p = (void *)((vm_addr_t)p - size);
if (!has_guard_pages(VMM_STACK | VMM_PER_THREAD) &&
DYNAMO_OPTION(stack_guard_pages)) {
alloc_size += PAGE_SIZE;
p = (byte *)p - PAGE_SIZE;
}
release_guarded_real_memory((vm_addr_t)p, alloc_size,
true , true,
VMM_STACK | VMM_PER_THREAD);
if (IF_DEBUG_ELSE(!dynamo_exited_log_and_stats, true))
RSTATS_SUB(stack_capacity, size);
}
* does not check any dstacks on the callback stack (win32) */
bool
is_stack_overflow(dcontext_t *dcontext, byte *sp)
{
* Currently sideline violates that for a thread stack, and we have separated
* -signal_stack_size, but all dstacks and d_r_initstack should be this size.
*/
byte *bottom = dcontext->dstack - DYNAMORIO_STACK_SIZE;
if (!DYNAMO_OPTION(stack_guard_pages) && !DYNAMO_OPTION(per_thread_guard_pages))
return false;
if (sp >= bottom - PAGE_SIZE && sp < bottom)
return true;
bottom = d_r_initstack - DYNAMORIO_STACK_SIZE;
if (sp >= bottom - PAGE_SIZE && sp < bottom)
return true;
return false;
}
byte *
d_r_map_file(file_t f, size_t *size INOUT, uint64 offs, app_pc addr, uint prot,
map_flags_t map_flags)
{
byte *view;
dynamo_vm_areas_lock();
view = os_map_file(f, size, offs, addr, prot, map_flags);
if (view != NULL) {
STATS_ADD_PEAK(file_map_capacity, *size);
account_for_memory((void *)view, *size, prot, true ,
true
_IF_DEBUG("map_file"));
}
dynamo_vm_areas_unlock();
return view;
}
bool
d_r_unmap_file(byte *map, size_t size)
{
bool success;
ASSERT(map != NULL && ALIGNED(map, PAGE_SIZE));
size = ALIGN_FORWARD(size, PAGE_SIZE);
dynamo_vm_areas_lock();
success = os_unmap_file(map, size);
if (success) {
* It should still be atomic to the outside observers.
*/
update_dynamo_areas_on_release(map, map + size, true );
STATS_SUB(file_map_capacity, size);
}
dynamo_vm_areas_unlock();
return success;
}
* circular dependencies bet vmareas and global heap (trust me, I've tried
* to support it with reentrant routines and recursive locks, the hard part
* is getting add_vm_area to be reentrant or to queue up adding areas,
* I think this solution is much more elegant, plus it avoids race conditions
* between DR memory allocation and the vmareas list by ensuring the list
* is up to date at the exact time of each query).
* Instead we on-demand walk the units.
* Freed units can usually be removed incrementally, except when we
* hold the heap_unit_lock when we run out of memory -- when we set
* a flag telling the caller of this routine to remove all heap areas
* from the vm list prior to calling us to add the real ones back in.
* Re-adding everyone is the simplest policy, so we don't have to keep
* track of who's been added.
* The caller is assumed to hold the dynamo vm areas write lock.
*/
void
heap_vmareas_synch_units()
{
heap_unit_t *u, *next;
* new unit in the course of adding these areas, so we use a recursive lock!
* furthermore, we need to own the global lock now, to avoid deadlock with
* another thread who does global_alloc and then needs a new unit!
* which means that the global_alloc lock must be recursive since vmareas
* may need to global_alloc...
*/
* always grab global_alloc first, then we won't have deadlocks
*/
acquire_recursive_lock(&global_alloc_lock);
acquire_recursive_lock(&heap_unit_lock);
if (dynamo_areas_pending_remove) {
dynamo_areas_pending_remove = false;
remove_dynamo_heap_areas();
* marked so. See case 4196.
*/
for (u = heapmgt->heap.units; u != NULL; u = u->next_global)
u->in_vmarea_list = false;
for (u = heapmgt->heap.dead; u != NULL; u = u->next_global)
u->in_vmarea_list = false;
}
for (u = heapmgt->heap.units; u != NULL; u = next) {
uint offs = has_guard_pages(u->which) ? (uint)PAGE_SIZE : 0;
app_pc start = (app_pc)u - offs;
* part of that PR we shouldn't have un-aligned anymore)
*/
app_pc end_align = (app_pc)ALIGN_FORWARD(UNIT_RESERVED_END(u), PAGE_SIZE);
app_pc end = end_align + offs;
next = u->next_global;
if (!u->in_vmarea_list &&
!is_vmm_reserved_address(start, end - start, NULL, NULL)) {
* may become dead if vector is resized, then u should be
* alive and u->next_global should be reset AFTER add */
bool next_may_die =
is_dynamo_area_buffer(UNIT_GET_START_PC(next));
* go to the dead unit list, or it can be released back to
* the OS. We'll mark it as being in vmarea list to avoid
* re-adding when going through dead one's, and we'll mark
* _before_ the potential free. If dynamo_areas.buf is
* freed back to the OS we'll have another iteration in
* update_dynamo_vm_areas() until we get fully
* synchronized, so we don't need to worry about the
* inconsistency.
*/
u->in_vmarea_list = true;
add_dynamo_heap_vm_area(start, end, true, false _IF_DEBUG("heap unit"));
* it, we may lose prot info about this unit.
* FIXME: Currently, this is done only at one place, which allocates unit
* as MEMPROT_READ | MEMPROT_WRITE. If other places are added, then this
* needs to change.
*/
update_all_memory_areas((app_pc)u, end_align, MEMPROT_READ | MEMPROT_WRITE,
DR_MEMTYPE_DATA);
if (offs != 0) {
update_all_memory_areas((app_pc)u - offs, (app_pc)u, MEMPROT_NONE,
DR_MEMTYPE_DATA);
update_all_memory_areas(end_align, end, MEMPROT_NONE, DR_MEMTYPE_DATA);
}
if (next_may_die) {
STATS_INC(num_vmareas_resize_synch);
* should still be live and point to the next live
*/
next = u->next_global;
}
}
}
for (u = heapmgt->heap.dead; u != NULL; u = next) {
uint offs = has_guard_pages(u->which) ? (uint)PAGE_SIZE : 0;
app_pc start = (app_pc)u - offs;
* part of that PR we shouldn't have un-aligned anymore)
*/
app_pc end_align = (app_pc)ALIGN_FORWARD(UNIT_RESERVED_END(u), PAGE_SIZE);
app_pc end = end_align + offs;
next = u->next_global;
if (!u->in_vmarea_list &&
!is_vmm_reserved_address(start, end - start, NULL, NULL)) {
u->in_vmarea_list = true;
add_dynamo_heap_vm_area(start, end, true, false _IF_DEBUG("dead heap unit"));
update_all_memory_areas((app_pc)u, end_align, MEMPROT_READ | MEMPROT_WRITE,
DR_MEMTYPE_DATA);
if (offs != 0) {
update_all_memory_areas(start, (app_pc)u, MEMPROT_NONE, DR_MEMTYPE_DATA);
update_all_memory_areas(end_align, end, MEMPROT_NONE, DR_MEMTYPE_DATA);
}
* dynamo_areas.buf vector, then next will no longer be a
* valid iterator over dead list
*/
if (is_dynamo_area_buffer(UNIT_GET_START_PC(next))) {
STATS_INC(num_vmareas_resize_synch);
ASSERT_NOT_TESTED();
next = u->next_global;
}
}
}
release_recursive_lock(&heap_unit_lock);
release_recursive_lock(&global_alloc_lock);
}
static void *
common_global_heap_alloc(thread_units_t *tu, size_t size HEAPACCT(which_heap_t which))
{
#ifdef STATIC_LIBRARY
if (standalone_library) {
* interoperability with tools like sanitizers.
* We limit this to static DR b/c we can have a direct call to libc malloc
* there and b/c that is the common use case for standalone mode these days.
*/
return malloc(size);
}
#endif
void *p;
acquire_recursive_lock(&global_alloc_lock);
p = common_heap_alloc(tu, size HEAPACCT(which));
release_recursive_lock(&global_alloc_lock);
if (p == NULL) {
* global alloc lock -- so we back out, grab it, and retry
*/
dynamo_vm_areas_lock();
acquire_recursive_lock(&global_alloc_lock);
p = common_heap_alloc(tu, size HEAPACCT(which));
release_recursive_lock(&global_alloc_lock);
dynamo_vm_areas_unlock();
}
ASSERT(p != NULL);
return p;
}
static void
common_global_heap_free(thread_units_t *tu, void *p,
size_t size HEAPACCT(which_heap_t which))
{
#ifdef STATIC_LIBRARY
if (standalone_library) {
* interoperability with tools like sanitizers.
* We limit this to static DR b/c we can have a direct call to libc malloc
* there and b/c that is the common use case for standalone mode these days.
*/
free(p);
return;
}
#endif
bool ok;
if (p == NULL) {
ASSERT(false && "attempt to free NULL");
return;
}
acquire_recursive_lock(&global_alloc_lock);
ok = common_heap_free(tu, p, size HEAPACCT(which));
release_recursive_lock(&global_alloc_lock);
if (!ok) {
* global alloc lock -- so we back out, grab it, and retry
*/
dynamo_vm_areas_lock();
acquire_recursive_lock(&global_alloc_lock);
ok = common_heap_free(tu, p, size HEAPACCT(which));
release_recursive_lock(&global_alloc_lock);
dynamo_vm_areas_unlock();
}
ASSERT(ok);
}
void *
global_heap_alloc(size_t size HEAPACCT(which_heap_t which))
{
void *p;
* regular DR library and not just drdecode, to support libraries that would use
* drdecode but that also have to work with full DR (i#2499).
*/
if (heapmgt == &temp_heapmgt &&
!heapmgt->global_heap_writable) {
* We need to develop a solution with atexit() or ELF destructors or sthg.
*/
standalone_init();
}
p = common_global_heap_alloc(&heapmgt->global_units, size HEAPACCT(which));
ASSERT(p != NULL);
LOG(GLOBAL, LOG_HEAP, 6, "\nglobal alloc: " PFX " (%d bytes)\n", p, size);
return p;
}
void
global_heap_free(void *p, size_t size HEAPACCT(which_heap_t which))
{
common_global_heap_free(&heapmgt->global_units, p, size HEAPACCT(which));
LOG(GLOBAL, LOG_HEAP, 6, "\nglobal free: " PFX " (%d bytes)\n", p, size);
}
allocates new_num elements of element_size
if ptr is NULL acts like global_heap_alloc,
copies an old_num elements of given size in the new area */
void *
global_heap_realloc(void *ptr, size_t old_num, size_t new_num,
size_t element_size HEAPACCT(which_heap_t which))
{
void *new_area = global_heap_alloc(new_num * element_size HEAPACCT(which));
if (ptr) {
memcpy(new_area, ptr, (old_num < new_num ? old_num : new_num) * element_size);
global_heap_free(ptr, old_num * element_size HEAPACCT(which));
}
return new_area;
}
* DYNAMO_OPTION(heap_commit_increment) is committed up front
*/
static heap_unit_t *
heap_create_unit(thread_units_t *tu, size_t size, bool must_be_new)
{
heap_unit_t *u = NULL, *dead = NULL, *prev_dead = NULL;
* units for oversized requests
*/
* DR area lock before heap_unit_lock
*/
ASSERT(safe_to_allocate_or_free_heap_units());
dynamo_vm_areas_lock();
acquire_recursive_lock(&heap_unit_lock);
* {next,prev}_global assignments too -- but need to know whether
* to re-protect -- do all at once, or each we need? add a writable
* flag to heap_unit_t?
*/
if (!must_be_new) {
for (dead = heapmgt->heap.dead; dead != NULL &&
(UNIT_RESERVED_SIZE(dead) < size || dead->which != tu->which);
prev_dead = dead, dead = dead->next_global)
;
}
if (dead != NULL) {
if (prev_dead == NULL)
heapmgt->heap.dead = dead->next_global;
else
prev_dead->next_global = dead->next_global;
u = dead;
heapmgt->heap.num_dead--;
RSTATS_DEC(heap_num_free);
release_recursive_lock(&heap_unit_lock);
LOG(GLOBAL, LOG_HEAP, 2,
"Re-using dead heap unit: " PFX "-" PFX " %d KB (need %d KB)\n", u,
((byte *)u) + size, UNIT_RESERVED_SIZE(u) / 1024, size / 1024);
} else {
size_t commit_size = DYNAMO_OPTION(heap_commit_increment);
release_recursive_lock(&heap_unit_lock);
* commit size w/o setting a lot of different unit size parameters.
*/
if (commit_size > size)
commit_size = size;
u = (heap_unit_t *)get_guarded_real_memory(size, commit_size,
MEMPROT_READ | MEMPROT_WRITE, false,
true, NULL, tu->which _IF_DEBUG(""));
ASSERT(u);
LOG(GLOBAL, LOG_HEAP, 2, "New heap unit: " PFX "-" PFX "\n", u,
((byte *)u) + size);
u->start_pc = (heap_pc)(((ptr_uint_t)u) + sizeof(heap_unit_t));
u->end_pc = ((heap_pc)u) + commit_size;
u->reserved_end_pc = ((heap_pc)u) + size;
u->in_vmarea_list = false;
u->which = tu->which;
RSTATS_ADD_PEAK(heap_capacity, commit_size);
STATS_ADD_PEAK(heap_reserved_only, (u->reserved_end_pc - u->end_pc));
}
RSTATS_ADD_PEAK(heap_num_live, 1);
ASSERT(u->which == tu->which);
u->cur_pc = u->start_pc;
u->next_local = NULL;
DODEBUG({
u->id = tu->num_units;
tu->num_units++;
});
acquire_recursive_lock(&heap_unit_lock);
u->next_global = heapmgt->heap.units;
if (heapmgt->heap.units != NULL)
heapmgt->heap.units->prev_global = u;
u->prev_global = NULL;
heapmgt->heap.units = u;
release_recursive_lock(&heap_unit_lock);
dynamo_vm_areas_unlock();
#ifdef DEBUG_MEMORY
DOCHECK(CHKLVL_MEMFILL,
memset(u->start_pc, HEAP_UNALLOCATED_BYTE, u->end_pc - u->start_pc););
#endif
return u;
}
static void
heap_free_unit(heap_unit_t *unit, dcontext_t *dcontext)
{
heap_unit_t *u, *prev_u;
#ifdef DEBUG_MEMORY
* frees and the free list cleanup, verify. */
* would include memory allocated ACCT_TOMBSTONE (which we don't currently
* use). Using ACCT_TOMBSTONE is dangerous since we will still free the
* unit here (say at proc or thread exit) even if there are ACCT_TOMBSTONE
* allocations in it. */
* not on the unit itself - FIXME: case 10434. Maybe we should embed the
* special heap unit header in the first special heap unit itself. */
DOCHECK(CHKLVL_MEMFILL, {
CLIENT_ASSERT(IF_HOTP(hotp_only_contains_leaked_trampoline(
unit->start_pc, unit->end_pc - unit->start_pc) ||)
* they don't always clean up. we have to relax here, but
* our threadunits_exit checks should find all leaks anyway.
*/
heapmgt->global_units.acct.cur_usage[ACCT_LIBDUP] > 0 ||
is_region_memset_to_char(unit->start_pc,
unit->end_pc - unit->start_pc,
HEAP_UNALLOCATED_BYTE)
* hard for Extension libs, etc. to clean up in such situations
*/
|| client_requested_exit,
"memory leak detected");
});
#endif
* DR area lock before heap_unit_lock
*/
ASSERT(safe_to_allocate_or_free_heap_units());
dynamo_vm_areas_lock();
acquire_recursive_lock(&heap_unit_lock);
* {next,prev}_global assignments too -- but need to know whether
* to re-protect -- do all at once, or each we need? add a writable
* flag to heap_unit_t?
*/
if (unit->prev_global != NULL) {
unit->prev_global->next_global = unit->next_global;
} else
heapmgt->heap.units = unit->next_global;
if (unit->next_global != NULL) {
unit->next_global->prev_global = unit->prev_global;
}
unit->prev_global = NULL;
RSTATS_DEC(heap_num_live);
* FIXME: share the policy with the fcache dead unit policy
* also, don't put special larger-than-max units on free list -- though
* we do now have support for doing so (after PR 415269)
*/
if (UNITALLOC(unit) <= HEAP_UNIT_MAX_SIZE &&
(heapmgt->heap.num_dead < 5 ||
heapmgt->heap.num_dead * 4U <= (uint)d_r_get_num_threads())) {
* when can take small and then needing to allocate when only
* have small left. Helps out with lots of small threads.
*/
for (u = heapmgt->heap.dead, prev_u = NULL;
u != NULL && UNIT_RESERVED_SIZE(u) < UNIT_RESERVED_SIZE(unit);
prev_u = u, u = u->next_global)
;
if (prev_u == NULL) {
unit->next_global = heapmgt->heap.dead;
heapmgt->heap.dead = unit;
} else {
unit->next_global = u;
prev_u->next_global = unit;
}
heapmgt->heap.num_dead++;
release_recursive_lock(&heap_unit_lock);
RSTATS_ADD_PEAK(heap_num_free, 1);
} else {
release_recursive_lock(&heap_unit_lock);
LOG(GLOBAL, LOG_HEAP, 1,
"\tfreeing excess dead unit " PFX "-" PFX " [-" PFX "]\n", unit,
UNIT_COMMIT_END(unit), UNIT_RESERVED_END(unit));
really_free_unit(unit);
}
dynamo_vm_areas_unlock();
}
#ifdef DEBUG_MEMORY
static heap_unit_t *
find_heap_unit(thread_units_t *tu, heap_pc p, size_t size)
{
* a new vmvector b/c of circular dependences. Proposal: use custom data
* field of vm_area_t in dynamo_areas list for heap entries to store a pointer
* to the heap_unit_t struct, and add a backpointer to the owning thread_units_t
* in heap_unit_t. Then have to make sure it's ok lock-wise to query the
* dynamo_areas in the middle of an alloc or a free. It should be but for
* global alloc and free we will have to grab the dynamo_areas lock up front
* every time instead of the very rare times now when we need a new unit.
*/
heap_unit_t *unit;
ASSERT(!POINTER_OVERFLOW_ON_ADD(p, size));
for (unit = tu->top_unit;
unit != NULL && (p < unit->start_pc || p + size > unit->end_pc);
unit = unit->next_local)
;
return unit;
}
#endif
static void
threadunits_init(dcontext_t *dcontext, thread_units_t *tu, size_t size, bool reachable)
{
int i;
DODEBUG({ tu->num_units = 0; });
tu->which = VMM_HEAP | (reachable ? VMM_REACHABLE : 0);
if (dcontext != GLOBAL_DCONTEXT) {
* save space for many-threaded apps. These units are rarely used.
* Note that this also precludes sharing dead units between thread-private
* and shared heaps due to the different "which" value.
*/
tu->which |= VMM_PER_THREAD;
}
tu->top_unit = heap_create_unit(tu, size, false );
tu->cur_unit = tu->top_unit;
tu->dcontext = dcontext;
tu->writable = true;
#ifdef HEAP_ACCOUNTING
memset(&tu->acct, 0, sizeof(tu->acct));
#endif
for (i = 0; i < BLOCK_TYPES; i++)
tu->free_list[i] = NULL;
}
#ifdef HEAP_ACCOUNTING
# define MAX_5_DIGIT 99999
static void
print_tu_heap_statistics(thread_units_t *tu, file_t logfile, const char *prefix)
{
int i;
size_t total = 0, cur = 0;
LOG(logfile, LOG_HEAP | LOG_STATS, 1, "%s heap breakdown:\n", prefix);
for (i = 0; i < ACCT_LAST; i++) {
LOG(logfile, LOG_HEAP | LOG_STATS, 1,
"%12s: cur=%5" SZFC "K, max=%5" SZFC "K, #=%7d, 1=", whichheap_name[i],
tu->acct.cur_usage[i] / 1024, tu->acct.max_usage[i] / 1024,
tu->acct.num_alloc[i]);
if (tu->acct.max_single[i] <= MAX_5_DIGIT)
LOG(logfile, LOG_HEAP | LOG_STATS, 1, "%5" SZFC, tu->acct.max_single[i]);
else {
LOG(logfile, LOG_HEAP | LOG_STATS, 1, "%4" SZFC "K",
tu->acct.max_single[i] / 1024);
}
LOG(logfile, LOG_HEAP | LOG_STATS, 1, ", new=%5" SZFC "K, re=%5" SZFC "K\n",
tu->acct.alloc_new[i] / 1024, tu->acct.alloc_reuse[i] / 1024);
total += tu->acct.max_usage[i];
cur += tu->acct.cur_usage[i];
}
LOG(logfile, LOG_HEAP | LOG_STATS, 1, "Total cur usage: %6" SZFC " KB\n", cur / 1024);
LOG(logfile, LOG_HEAP | LOG_STATS, 1,
"Total max (not nec. all used simult.): %6" SZFC " KB\n", total / 1024);
}
void
print_heap_statistics()
{
dcontext_t *dcontext = get_thread_private_dcontext();
DOSTATS({
uint i;
LOG(GLOBAL, LOG_STATS, 1, "Heap bucket usage counts and wasted memory:\n");
for (i = 0; i < BLOCK_TYPES; i++) {
LOG(GLOBAL, LOG_STATS | LOG_HEAP, 1,
"%2d %3d count=%9u peak_count=%9u peak_wasted=%9u peak_align=%9u\n", i,
BLOCK_SIZES[i], block_total_count[i], block_peak_count[i],
block_peak_wasted[i], block_peak_align_pad[i]);
}
});
if (dcontext != NULL) {
thread_heap_t *th = (thread_heap_t *)dcontext->heap_field;
if (th != NULL) {
print_tu_heap_statistics(th->local_heap, THREAD, "Thread");
ASSERT(th->nonpersistent_heap != NULL);
print_tu_heap_statistics(th->nonpersistent_heap, THREAD,
"Thread non-persistent");
if (!REACHABLE_HEAP()) {
ASSERT(th->reachable_heap != NULL);
print_tu_heap_statistics(th->reachable_heap, THREAD, "Thread reachable");
}
}
}
print_tu_heap_statistics(&heapmgt->global_nonpersistent_units, GLOBAL,
"Non-persistent global units");
if (!REACHABLE_HEAP()) {
print_tu_heap_statistics(&heapmgt->global_reachable_units, GLOBAL,
"Reachable global units");
}
print_tu_heap_statistics(&heapmgt->global_unprotected_units, GLOBAL,
"Uprotected global units");
print_tu_heap_statistics(&global_racy_units, GLOBAL, "Racy Up-to-date Process");
print_tu_heap_statistics(&heapmgt->global_units, GLOBAL,
"Updated-at-end Process (max is total of maxes)");
}
static void
add_heapacct_to_global_stats(heap_acct_t *acct)
{
* FIXME: this gives a nice in-one-place total, but loses the
* global-heap-only stats -- perhaps should add a total_units stats
* to capture total and leave global alone here?
*/
uint i;
acquire_recursive_lock(&global_alloc_lock);
for (i = 0; i < ACCT_LAST; i++) {
heapmgt->global_units.acct.alloc_reuse[i] += acct->alloc_reuse[i];
heapmgt->global_units.acct.alloc_new[i] += acct->alloc_new[i];
heapmgt->global_units.acct.cur_usage[i] += acct->cur_usage[i];
heapmgt->global_units.acct.max_usage[i] += acct->max_usage[i];
heapmgt->global_units.acct.max_single[i] += acct->max_single[i];
heapmgt->global_units.acct.num_alloc[i] += acct->num_alloc[i];
}
release_recursive_lock(&global_alloc_lock);
}
#endif
static void
threadunits_exit(thread_units_t *tu, dcontext_t *dcontext)
{
heap_unit_t *u, *next_u;
#ifdef DEBUG
size_t total_heap_used = 0;
# ifdef HEAP_ACCOUNTING
int j;
# endif
#endif
#ifdef DEBUG_MEMORY
uint i;
for (i = 0; i < BLOCK_TYPES; i++) {
heap_pc p, next_p;
for (p = tu->free_list[i]; p != NULL; p = next_p) {
next_p = *(heap_pc *)p;
*(heap_pc *)p = (heap_pc)HEAP_UNALLOCATED_PTR_UINT;
DOCHECK(CHKLVL_MEMFILL, {
if (i < BLOCK_TYPES - 1) {
CLIENT_ASSERT(is_region_memset_to_char(p, BLOCK_SIZES[i],
HEAP_UNALLOCATED_BYTE),
"memory corruption detected");
} else {
CLIENT_ASSERT(is_region_memset_to_char(p, VARIABLE_SIZE(p),
HEAP_UNALLOCATED_BYTE),
"memory corruption detected");
MEMSET_HEADER(p, HEAP_UNALLOCATED);
}
});
}
tu->free_list[i] = NULL;
}
#endif
u = tu->top_unit;
while (u != NULL) {
DOLOG(1, LOG_HEAP | LOG_STATS, {
size_t num_used = u->cur_pc - u->start_pc;
total_heap_used += num_used;
LOG(THREAD, LOG_HEAP | LOG_STATS, 1,
"Heap unit %d @" PFX "-" PFX " [-" PFX "] (" SZFMT " [/" SZFMT
"] KB): used " SZFMT " bytes\n",
u->id, u, UNIT_COMMIT_END(u), UNIT_RESERVED_END(u),
(UNIT_COMMIT_SIZE(u)) / 1024, (UNIT_RESERVED_SIZE(u)) / 1024, num_used);
});
next_u = u->next_local;
heap_free_unit(u, dcontext);
u = next_u;
}
LOG(THREAD, LOG_HEAP | LOG_STATS, 1, "\tTotal heap used: " SZFMT " KB\n",
total_heap_used / 1024);
#if defined(DEBUG) && defined(HEAP_ACCOUNTING)
for (j = 0; j < ACCT_LAST; j++) {
size_t usage = tu->acct.cur_usage[j];
if (usage > 0) {
LOG(THREAD, LOG_HEAP | LOG_STATS, 1,
"WARNING: %s " SZFMT " bytes not freed!\n", whichheap_name[j],
tu->acct.cur_usage[j]);
# ifdef HOT_PATCHING_INTERFACE
if (DYNAMO_OPTION(hotp_only) &&
((j == ACCT_SPECIAL && usage == (size_t)hotp_only_tramp_bytes_leaked) ||
(j == ACCT_MEM_MGT &&
usage == (size_t)(get_special_heap_header_size() + 4) &&
hotp_only_tramp_bytes_leaked > 0)))
continue;
# endif
if (j != ACCT_TOMBSTONE &&
* they don't always clean up
*/
j != ACCT_LIBDUP && INTERNAL_OPTION(heap_accounting_assert)) {
SYSLOG_INTERNAL_ERROR("memory leak: %s " SZFMT " bytes not freed",
whichheap_name[j], tu->acct.cur_usage[j]);
* hard for Extension libs, etc. to clean up in such situations:
*/
CLIENT_ASSERT(client_requested_exit || false, "memory leak detected");
}
}
}
if (tu != &heapmgt->global_units)
add_heapacct_to_global_stats(&tu->acct);
DOLOG(1, LOG_HEAP | LOG_STATS, {
print_tu_heap_statistics(tu, THREAD,
dcontext == GLOBAL_DCONTEXT ? "Process" : "Thread");
});
#endif
}
void
heap_thread_reset_init(dcontext_t *dcontext)
{
thread_heap_t *th = (thread_heap_t *)dcontext->heap_field;
ASSERT(th->nonpersistent_heap != NULL);
threadunits_init(dcontext, th->nonpersistent_heap,
DYNAMO_OPTION(initial_heap_nonpers_size), false);
}
void
heap_thread_init(dcontext_t *dcontext)
{
thread_heap_t *th =
(thread_heap_t *)global_heap_alloc(sizeof(thread_heap_t) HEAPACCT(ACCT_MEM_MGT));
dcontext->heap_field = (void *)th;
th->local_heap = (thread_units_t *)global_heap_alloc(sizeof(thread_units_t)
HEAPACCT(ACCT_MEM_MGT));
threadunits_init(dcontext, th->local_heap, HEAP_UNIT_MIN_SIZE, false);
th->nonpersistent_heap = (thread_units_t *)global_heap_alloc(
sizeof(thread_units_t) HEAPACCT(ACCT_MEM_MGT));
if (!REACHABLE_HEAP()) {
th->reachable_heap = (thread_units_t *)global_heap_alloc(
sizeof(thread_units_t) HEAPACCT(ACCT_MEM_MGT));
threadunits_init(dcontext, th->reachable_heap, HEAP_UNIT_MIN_SIZE, true);
} else
th->reachable_heap = NULL;
heap_thread_reset_init(dcontext);
#ifdef UNIX
th->fork_copy_start = NULL;
th->fork_copy_size = 0;
#endif
}
void
heap_thread_reset_free(dcontext_t *dcontext)
{
thread_heap_t *th = (thread_heap_t *)dcontext->heap_field;
ASSERT(th->nonpersistent_heap != NULL);
* heap_reset_free() to free!
* FIXME: for reset, don't free last unit so don't have to
* recreate in reset_init()
*/
threadunits_exit(th->nonpersistent_heap, dcontext);
}
void
heap_thread_exit(dcontext_t *dcontext)
{
thread_heap_t *th = (thread_heap_t *)dcontext->heap_field;
threadunits_exit(th->local_heap, dcontext);
heap_thread_reset_free(dcontext);
global_heap_free(th->local_heap, sizeof(thread_units_t) HEAPACCT(ACCT_MEM_MGT));
ASSERT(th->nonpersistent_heap != NULL);
global_heap_free(th->nonpersistent_heap,
sizeof(thread_units_t) HEAPACCT(ACCT_MEM_MGT));
if (!REACHABLE_HEAP()) {
ASSERT(th->reachable_heap != NULL);
threadunits_exit(th->reachable_heap, dcontext);
global_heap_free(th->reachable_heap,
sizeof(thread_units_t) HEAPACCT(ACCT_MEM_MGT));
}
global_heap_free(th, sizeof(thread_heap_t) HEAPACCT(ACCT_MEM_MGT));
}
#if defined(DEBUG_MEMORY) && defined(DEBUG)
void
print_free_list(thread_units_t *tu, int i)
{
void *p;
int len = 0;
dcontext_t *dcontext = tu->dcontext;
LOG(THREAD, LOG_HEAP, 1, "Free list for size %d (== %d bytes):\n", i, BLOCK_SIZES[i]);
p = (void *)tu->free_list[i];
while (p != NULL) {
LOG(THREAD, LOG_HEAP, 1, "\tp = " PFX "\n", p);
len++;
p = *((char **)p);
}
LOG(THREAD, LOG_HEAP, 1, "Total length is %d\n", len);
}
#endif
* Returns the amount it increased the unit by, so caller should increment
* end_pc.
* Both end_pc and reserved_end_pc are assumed to be open-ended!
*/
static size_t
common_heap_extend_commitment(heap_pc cur_pc, heap_pc end_pc, heap_pc reserved_end_pc,
size_t size_need, uint prot, which_vmm_t which)
{
if (end_pc < reserved_end_pc && !POINTER_OVERFLOW_ON_ADD(cur_pc, size_need)) {
size_t commit_size = DYNAMO_OPTION(heap_commit_increment);
if (POINTER_OVERFLOW_ON_ADD(end_pc, commit_size))
return 0;
if (cur_pc + size_need > end_pc + commit_size) {
commit_size =
ALIGN_FORWARD(cur_pc + size_need - (ptr_uint_t)end_pc, PAGE_SIZE);
}
if (end_pc + commit_size > reserved_end_pc ||
POINTER_OVERFLOW_ON_ADD(end_pc,
commit_size) ) {
* we keep an invariant that all units but the current one
* are fully committed, so our algorithm for looking at the end
* of prior units holds
*/
commit_size = reserved_end_pc - end_pc;
}
ASSERT(!POINTER_OVERFLOW_ON_ADD(end_pc, commit_size) &&
end_pc + commit_size <= reserved_end_pc);
extend_commitment(end_pc, commit_size, prot, false , which);
#ifdef DEBUG_MEMORY
memset(vmcode_get_writable_addr(end_pc), HEAP_UNALLOCATED_BYTE, commit_size);
#endif
RSTATS_ADD_PEAK(heap_capacity, commit_size);
STATS_SUB(heap_reserved_only, commit_size);
ASSERT(end_pc <= reserved_end_pc);
return commit_size;
} else
return 0;
}
static void
heap_unit_extend_commitment(heap_unit_t *u, size_t size_need, uint prot)
{
u->end_pc += common_heap_extend_commitment(u->cur_pc, u->end_pc, u->reserved_end_pc,
size_need, prot, u->which);
}
* returns NULL iff caller needs to grab dynamo_vm_areas_lock() and retry
*/
static void *
common_heap_alloc(thread_units_t *tu, size_t size HEAPACCT(which_heap_t which))
{
heap_unit_t *u = tu->cur_unit;
heap_pc p = NULL;
int bucket = 0;
size_t alloc_size, aligned_size;
#if defined(DEBUG_MEMORY) && defined(DEBUG)
size_t check_alloc_size;
dcontext_t *dcontext = tu->dcontext;
* can have a prohibitive perf cost!
*/
uint chklvl = CHKLVL_MEMFILL + (IF_HEAPACCT_ELSE(which == ACCT_LIBDUP ? 1 : 0, 0));
ASSERT_CURIOSITY(which != ACCT_TOMBSTONE &&
"Do you really need to use ACCT_TOMBSTONE? (potentially dangerous)");
#endif
ASSERT(size > 0);
ASSERT(size < MAX_VALID_HEAP_ALLOCATION && "potential integer overflow");
if (size > MAX_VALID_HEAP_ALLOCATION) {
* overflows sizes up to UINT_MAX - sizeof(heap_unit_t), but
* INT_MAX should be more than enough.
*
* Caller will likely crash, but that is better than a heap
* overflow, where a crash would be the best we can hope for.
*/
return NULL;
}
* which bucket is used. */
aligned_size = ALIGN_FORWARD(size, HEAP_ALIGNMENT);
while (aligned_size > BLOCK_SIZES[bucket])
bucket++;
if (bucket == BLOCK_TYPES - 1)
alloc_size = aligned_size + HEADER_SIZE;
else
alloc_size = BLOCK_SIZES[bucket];
ASSERT(size <= alloc_size);
#ifdef DEBUG_MEMORY
check_alloc_size = alloc_size;
#endif
if (alloc_size > MAXROOM) {
heap_unit_t *new_unit, *prev;
size_t unit_size = ALIGN_FORWARD(size + sizeof(heap_unit_t), PAGE_SIZE);
ASSERT(size < unit_size && "overflow");
if (!safe_to_allocate_or_free_heap_units()) {
* global alloc lock -- so we back out, grab it, and then come back
*/
return NULL;
}
* excess size until this is freed and put back on dead list.
* (Currently we don't put oversized units on dead list though.)
*/
new_unit = heap_create_unit(tu, unit_size, false );
heap_unit_extend_commitment(new_unit, size, MEMPROT_READ | MEMPROT_WRITE);
prev = tu->top_unit;
alloc_size = size;
* where it is)
*/
while (prev != u && prev->next_local != u) {
ASSERT(prev != NULL && prev->next_local != NULL);
prev = prev->next_local;
}
if (prev == u) {
ASSERT(prev == tu->top_unit);
tu->top_unit = new_unit;
} else
prev->next_local = new_unit;
new_unit->next_local = u;
#ifdef DEBUG_MEMORY
LOG(THREAD, LOG_HEAP, 3, "\tCreating new oversized heap unit %d (%d [/%d] KB)\n",
new_unit->id, UNIT_COMMIT_SIZE(new_unit) / 1024,
UNIT_RESERVED_SIZE(new_unit) / 1024);
#endif
p = new_unit->start_pc;
new_unit->cur_pc += size;
ACCOUNT_FOR_ALLOC(alloc_new, tu, which, size, size);
goto done_allocating;
}
if (tu->free_list[bucket] != NULL) {
if (bucket == BLOCK_TYPES - 1) {
size_t sz;
heap_pc next = tu->free_list[bucket];
heap_pc prev;
do {
prev = p;
p = next;
sz = VARIABLE_SIZE(next);
next = *((heap_pc *)p);
} while (aligned_size > sz && next != NULL);
if (aligned_size <= sz) {
ASSERT(ALIGNED(next, HEAP_ALIGNMENT));
if (p == tu->free_list[bucket])
tu->free_list[bucket] = next;
else
*((heap_pc *)prev) = next;
#ifdef DEBUG_MEMORY
LOG(THREAD, LOG_HEAP, 2,
"Variable-size block: allocating " PFX " (%d bytes [%d aligned] in "
"%d block)\n",
p, size, aligned_size, sz);
DOCHECK(CHKLVL_DEFAULT, {
ASSERT(find_heap_unit(tu, p, sz) != NULL);
});
#endif
ASSERT(ALIGNED(sz, HEAP_ALIGNMENT));
alloc_size = sz + HEADER_SIZE;
ACCOUNT_FOR_ALLOC(alloc_reuse, tu, which, alloc_size, aligned_size);
} else {
p = NULL;
}
} else {
p = tu->free_list[bucket];
tu->free_list[bucket] = *((heap_pc *)p);
ASSERT(ALIGNED(tu->free_list[bucket], HEAP_ALIGNMENT));
#ifdef DEBUG_MEMORY
DOCHECK(CHKLVL_DEFAULT, {
ASSERT(find_heap_unit(tu, p, alloc_size) != NULL);
});
#endif
ACCOUNT_FOR_ALLOC(alloc_reuse, tu, which, alloc_size, aligned_size);
}
}
if (p == NULL) {
* should use the larger blocks instead of failing! */
ASSERT(ALIGNED(u->cur_pc, HEAP_ALIGNMENT));
ASSERT(ALIGNED(alloc_size, HEAP_ALIGNMENT));
if (u->cur_pc + alloc_size > u->end_pc ||
POINTER_OVERFLOW_ON_ADD(u->cur_pc, alloc_size) ) {
* allocate a new unit. */
if (!safe_to_allocate_or_free_heap_units()) {
* lock before global alloc lock in case we end up adding a new
* unit or we hit oom (which may free units) while extending
* the commmitment -- so we back out, grab it, and then come
* back. */
return NULL;
}
heap_unit_extend_commitment(u, alloc_size, MEMPROT_READ | MEMPROT_WRITE);
if (u->cur_pc + alloc_size > u->end_pc ||
POINTER_OVERFLOW_ON_ADD(u->cur_pc, alloc_size) ) {
* FIXME: instead should put end of unit space on free list!
*/
heap_unit_t *prev = tu->top_unit;
while (1) {
* who though they may have extra space from alignment
* may be freed wholesale when primary alloc is freed
*/
if (UNITALLOC(prev) <= HEAP_UNIT_MAX_SIZE &&
!POINTER_OVERFLOW_ON_ADD(prev->cur_pc, alloc_size) &&
prev->cur_pc + alloc_size <= prev->end_pc) {
tu->cur_unit = prev;
u = prev;
break;
}
if (prev->next_local == NULL) {
* double size of old unit (until hit max size)
*/
heap_unit_t *new_unit;
size_t unit_size;
unit_size = UNITALLOC(u) * 2;
while (unit_size < alloc_size + UNITOVERHEAD)
unit_size *= 2;
if (unit_size > HEAP_UNIT_MAX_SIZE)
unit_size = HEAP_UNIT_MAX_SIZE;
ASSERT(unit_size > UNITOVERHEAD);
new_unit = heap_create_unit(tu, unit_size, false );
prev->next_local = new_unit;
#ifdef DEBUG_MEMORY
LOG(THREAD, LOG_HEAP, 2,
"\tCreating new heap unit %d (%d [/%d] KB)\n", new_unit->id,
UNIT_COMMIT_SIZE(new_unit) / 1024,
UNIT_RESERVED_SIZE(new_unit) / 1024);
#endif
* we'll try to use the free room at the end of the old unit(s)
* only when we next run out of room
*/
tu->cur_unit = new_unit;
u = new_unit;
heap_unit_extend_commitment(u, alloc_size,
MEMPROT_READ | MEMPROT_WRITE);
ASSERT(alloc_size <= (ptr_uint_t)(u->end_pc - u->cur_pc));
break;
}
prev = prev->next_local;
}
}
}
p = u->cur_pc;
if (bucket == BLOCK_TYPES - 1) {
p += HEADER_SIZE;
VARIABLE_SIZE(p) = aligned_size;
}
u->cur_pc += alloc_size;
ACCOUNT_FOR_ALLOC(alloc_new, tu, which, alloc_size, aligned_size);
}
DOSTATS({
ATOMIC_ADD(int, block_count[bucket], 1);
ATOMIC_ADD(int, block_total_count[bucket], 1);
ATOMIC_MAX(int, block_peak_count[bucket], block_count[bucket]);
ASSERT(CHECK_TRUNCATE_TYPE_uint(alloc_size - aligned_size));
ATOMIC_ADD(int, block_wasted[bucket], (int)(alloc_size - aligned_size));
ATOMIC_MAX(int, block_peak_wasted[bucket], block_wasted[bucket]);
if (aligned_size > size) {
ASSERT(CHECK_TRUNCATE_TYPE_uint(aligned_size - size));
ATOMIC_ADD(int, block_align_pad[bucket], (int)(aligned_size - size));
ATOMIC_MAX(int, block_peak_align_pad[bucket], block_align_pad[bucket]);
STATS_ADD_PEAK(heap_align, aligned_size - size);
LOG(GLOBAL, LOG_STATS, 5,
"alignment mismatch: %s ask %d, aligned is %d -> %d pad\n",
IF_HEAPACCT_ELSE(whichheap_name[which], ""), size, aligned_size,
aligned_size - size);
}
if (bucket == BLOCK_TYPES - 1) {
STATS_ADD(heap_headers, HEADER_SIZE);
STATS_INC(heap_allocs_variable);
} else {
STATS_INC(heap_allocs_buckets);
if (alloc_size > aligned_size) {
STATS_ADD_PEAK(heap_bucket_pad, alloc_size - aligned_size);
LOG(GLOBAL, LOG_STATS, 5,
"bucket mismatch: %s ask (aligned) %d, got %d, -> %d\n",
IF_HEAPACCT_ELSE(whichheap_name[which], ""), aligned_size, alloc_size,
alloc_size - aligned_size);
}
}
});
done_allocating:
#ifdef DEBUG_MEMORY
if (bucket == BLOCK_TYPES - 1 && check_alloc_size <= MAXROOM) {
DOCHECK(chklvl, {
CLIENT_ASSERT(
is_region_memset_to_char(p + sizeof(heap_pc *),
(alloc_size - HEADER_SIZE) - sizeof(heap_pc *),
HEAP_UNALLOCATED_BYTE),
"memory corruption detected");
});
LOG(THREAD, LOG_HEAP, 6,
"\nalloc var " PFX "-" PFX " %d bytes, ret " PFX "-" PFX " %d bytes\n",
p - HEADER_SIZE, p - HEADER_SIZE + alloc_size, alloc_size, p, p + size, size);
DOCHECK(chklvl,
memset(p + size, HEAP_PAD_BYTE, (alloc_size - HEADER_SIZE) - size););
} else {
DOCHECK(chklvl, {
CLIENT_ASSERT(is_region_memset_to_char(p + sizeof(heap_pc *),
alloc_size - sizeof(heap_pc *),
HEAP_UNALLOCATED_BYTE),
"memory corruption detected");
});
LOG(THREAD, LOG_HEAP, 6,
"\nalloc fix or oversize " PFX "-" PFX " %d bytes, ret " PFX "-" PFX
" %d bytes\n",
p, p + alloc_size, alloc_size, p, p + size, size);
DOCHECK(chklvl, memset(p + size, HEAP_PAD_BYTE, alloc_size - size););
}
DOCHECK(chklvl, memset(p, HEAP_ALLOCATED_BYTE, size););
# ifdef HEAP_ACCOUNTING
LOG(THREAD, LOG_HEAP, 6, "\t%s\n", whichheap_name[which]);
# endif
#endif
return (void *)p;
}
void *
heap_alloc(dcontext_t *dcontext, size_t size HEAPACCT(which_heap_t which))
{
thread_units_t *tu;
void *ret_val;
if (dcontext == GLOBAL_DCONTEXT)
return global_heap_alloc(size HEAPACCT(which));
tu = ((thread_heap_t *)dcontext->heap_field)->local_heap;
ret_val = common_heap_alloc(tu, size HEAPACCT(which));
ASSERT(ret_val != NULL);
return ret_val;
}
* returns false if caller needs to grab dynamo_vm_areas_lock() and retry
*/
static bool
common_heap_free(thread_units_t *tu, void *p_void,
size_t size HEAPACCT(which_heap_t which))
{
int bucket = 0;
heap_pc p = (heap_pc)p_void;
#if defined(DEBUG) && (defined(DEBUG_MEMORY) || defined(HEAP_ACCOUNTING))
dcontext_t *dcontext = tu->dcontext;
* can have a prohibitive perf cost!
* XXX: b/c of re-use we have to memset on free. Perhaps we should
* have a separate heap pool for private libs. But, the overhead
* from that final memset is small compared to what we've already
* saved, so maybe not worth it.
*/
uint chklvl = CHKLVL_MEMFILL + (IF_HEAPACCT_ELSE(which == ACCT_LIBDUP ? 1 : 0, 0));
#endif
size_t alloc_size, aligned_size = ALIGN_FORWARD(size, HEAP_ALIGNMENT);
ASSERT(size > 0);
ASSERT(p != NULL);
#ifdef DEBUG_MEMORY
* perform a double free, but it can fire if we ever free a data structure
* that has the 0xcdcdcdcd bitpattern in the first or last 4 bytes. This
* has happened a few times:
*
* - case 8802: App's eax is 0xcdcdcdcd (from an app dbg memset) and we have
* dcontext->allocated_start==dcontext.
* - i#417: On Linux x64 we get rax == 0xcdcdcdcd from a memset, and
* opnd_create_reg() only updates part of the register before returning by
* value in RAX:RDX. We initialize to zero in debug mode to work around
* this.
* - i#540: On Win7 x64 we see this assert when running the TSan tests in
* NegativeTests.WindowsRegisterWaitForSingleObjectTest.
*
* For now, we've downgraded this to a curiosity, but if it fires too much
* in the future we should maintain a separate data structure in debug mode
* to perform this check. We accept objects that start with 0xcdcdcdcd so
* long as the second four bytes are not also 0xcdcdcdcd.
*/
DOCHECK(chklvl, {
ASSERT_CURIOSITY(
(*(uint *)p != HEAP_UNALLOCATED_UINT ||
(size >= 2 * sizeof(uint) && *(((uint *)p) + 1) != HEAP_UNALLOCATED_UINT)) &&
*(uint *)(p + size - sizeof(int)) != HEAP_UNALLOCATED_UINT &&
"attempting to free memory containing HEAP_UNALLOCATED pattern, "
"possible double free!");
});
#endif
while (aligned_size > BLOCK_SIZES[bucket])
bucket++;
if (bucket == BLOCK_TYPES - 1)
alloc_size = aligned_size + HEADER_SIZE;
else
alloc_size = BLOCK_SIZES[bucket];
if (alloc_size > MAXROOM) {
heap_unit_t *u = tu->top_unit, *prev = NULL;
#ifdef DEBUG_MEMORY
DOCHECK(CHKLVL_DEFAULT, {
ASSERT(find_heap_unit(tu, p, size) != NULL);
});
#endif
if (!safe_to_allocate_or_free_heap_units()) {
* global alloc lock -- so we back out, grab it, and then come back
*/
return false;
}
while (u != NULL && u->start_pc != p) {
prev = u;
u = u->next_local;
}
ASSERT(u != NULL);
* for future use -- no problems will be caused by it being
* larger than normal
*/
if (prev == NULL)
tu->top_unit = u->next_local;
else
prev->next_local = u->next_local;
#ifdef DEBUG_MEMORY
LOG(THREAD, LOG_HEAP, 3, "\tFreeing oversized heap unit %d (%d KB)\n", u->id,
size / 1024);
* the unit, is needed for an assert in heap_free_unit anyways */
DOCHECK(CHKLVL_MEMFILL, memset(p, HEAP_UNALLOCATED_BYTE, size););
#endif
ASSERT(size <= UNITROOM(u));
heap_free_unit(u, tu->dcontext);
ACCOUNT_FOR_FREE(tu, which, size);
return true;
} else if (bucket == BLOCK_TYPES - 1) {
ASSERT(GET_VARIABLE_ALLOCATION_SIZE(p) >= alloc_size);
alloc_size = GET_VARIABLE_ALLOCATION_SIZE(p);
ASSERT(alloc_size - HEADER_SIZE >= aligned_size);
}
#if defined(DEBUG) || defined(DEBUG_MEMORY) || defined(HEAP_ACCOUNTING)
if (bucket == BLOCK_TYPES - 1) {
# ifdef DEBUG_MEMORY
LOG(THREAD, LOG_HEAP, 6,
"\nfree var " PFX "-" PFX " %d bytes, asked " PFX "-" PFX " %d bytes\n",
p - HEADER_SIZE, p - HEADER_SIZE + alloc_size, alloc_size, p, p + size, size);
ASSERT_MESSAGE(chklvl, "heap overflow",
is_region_memset_to_char(
p + size, (alloc_size - HEADER_SIZE) - size, HEAP_PAD_BYTE));
DOCHECK(CHKLVL_DEFAULT,
{
ASSERT(find_heap_unit(tu, p, alloc_size - HEADER_SIZE) != NULL);
});
DOCHECK(CHKLVL_MEMFILL,
memset(p, HEAP_UNALLOCATED_BYTE, alloc_size - HEADER_SIZE););
# endif
STATS_SUB(heap_headers, HEADER_SIZE);
} else {
# ifdef DEBUG_MEMORY
LOG(THREAD, LOG_HEAP, 6,
"\nfree fix " PFX "-" PFX " %d bytes, asked " PFX "-" PFX " %d bytes\n", p,
p + alloc_size, alloc_size, p, p + size, size);
ASSERT_MESSAGE(
chklvl, "heap overflow",
is_region_memset_to_char(p + size, alloc_size - size, HEAP_PAD_BYTE));
DOCHECK(CHKLVL_DEFAULT, {
ASSERT(find_heap_unit(tu, p, alloc_size) != NULL);
});
DOCHECK(CHKLVL_MEMFILL, memset(p, HEAP_UNALLOCATED_BYTE, alloc_size););
# endif
STATS_SUB(heap_bucket_pad, (alloc_size - aligned_size));
}
STATS_SUB(heap_align, (aligned_size - size));
DOSTATS({
ATOMIC_ADD(int, block_count[bucket], -1);
ATOMIC_ADD(int, block_wasted[bucket], -(int)(alloc_size - aligned_size));
ATOMIC_ADD(int, block_align_pad[bucket], -(int)(aligned_size - size));
});
# ifdef HEAP_ACCOUNTING
LOG(THREAD, LOG_HEAP, 6, "\t%s\n", whichheap_name[which]);
ACCOUNT_FOR_FREE(tu, which, alloc_size);
# endif
#endif
*((heap_pc *)p) = tu->free_list[bucket];
ASSERT(ALIGNED(tu->free_list[bucket], HEAP_ALIGNMENT));
tu->free_list[bucket] = p;
ASSERT(ALIGNED(tu->free_list[bucket], HEAP_ALIGNMENT));
return true;
}
void
heap_free(dcontext_t *dcontext, void *p, size_t size HEAPACCT(which_heap_t which))
{
thread_units_t *tu;
DEBUG_DECLARE(bool ok;)
if (dcontext == GLOBAL_DCONTEXT) {
global_heap_free(p, size HEAPACCT(which));
return;
}
tu = ((thread_heap_t *)dcontext->heap_field)->local_heap;
DEBUG_DECLARE(ok =) common_heap_free(tu, p, size HEAPACCT(which));
ASSERT(ok);
}
bool
local_heap_protected(dcontext_t *dcontext)
{
thread_heap_t *th = (thread_heap_t *)dcontext->heap_field;
return (!th->local_heap->writable ||
(th->nonpersistent_heap != NULL && !th->nonpersistent_heap->writable) ||
(th->reachable_heap != NULL && !th->reachable_heap->writable));
}
static inline void
protect_local_units_helper(heap_unit_t *u, bool writable)
{
* were allocated with separate calls so we don't try to combine
* adjacent units here
*/
while (u != NULL) {
change_protection(UNIT_ALLOC_START(u), UNIT_COMMIT_SIZE(u), writable);
u = u->next_local;
}
}
static void
protect_threadunits(thread_units_t *tu, bool writable)
{
ASSERT(TEST(SELFPROT_LOCAL, dynamo_options.protect_mask));
if (tu->writable == writable)
return;
protect_local_units_helper(tu->top_unit, writable);
tu->writable = writable;
}
void
protect_local_heap(dcontext_t *dcontext, bool writable)
{
thread_heap_t *th = (thread_heap_t *)dcontext->heap_field;
protect_threadunits(th->local_heap, writable);
protect_threadunits(th->nonpersistent_heap, writable);
if (!REACHABLE_HEAP())
protect_threadunits(th->reachable_heap, writable);
}
* are protected by us here, so ignore os heap
*/
void
protect_global_heap(bool writable)
{
ASSERT(TEST(SELFPROT_GLOBAL, dynamo_options.protect_mask));
acquire_recursive_lock(&global_alloc_lock);
if (heapmgt->global_heap_writable == writable) {
release_recursive_lock(&global_alloc_lock);
return;
}
* were allocated with separate calls so we don't try to combine
* adjacent units here
* FIXME: That may no longer be true for our virtual memory manager that
* will in fact be allocated as a single unit. It is only in case
* we have run out of that initial allocation that we may have to
* keep a separate list of allocations.
*/
if (!writable) {
ASSERT(heapmgt->global_heap_writable);
heapmgt->global_heap_writable = writable;
}
protect_local_units_helper(heapmgt->global_units.top_unit, writable);
protect_local_units_helper(heapmgt->global_nonpersistent_units.top_unit, writable);
if (writable) {
ASSERT(!heapmgt->global_heap_writable);
heapmgt->global_heap_writable = writable;
}
release_recursive_lock(&global_alloc_lock);
}
* versions except the unit used
*/
void *
global_unprotected_heap_alloc(size_t size HEAPACCT(which_heap_t which))
{
void *p = common_global_heap_alloc(&heapmgt->global_unprotected_units,
size HEAPACCT(which));
ASSERT(p != NULL);
LOG(GLOBAL, LOG_HEAP, 6, "\nglobal unprotected alloc: " PFX " (%d bytes)\n", p, size);
return p;
}
void
global_unprotected_heap_free(void *p, size_t size HEAPACCT(which_heap_t which))
{
common_global_heap_free(&heapmgt->global_unprotected_units, p, size HEAPACCT(which));
LOG(GLOBAL, LOG_HEAP, 6, "\nglobal unprotected free: " PFX " (%d bytes)\n", p, size);
}
void *
nonpersistent_heap_alloc(dcontext_t *dcontext, size_t size HEAPACCT(which_heap_t which))
{
void *p;
if (dcontext == GLOBAL_DCONTEXT) {
p = common_global_heap_alloc(&heapmgt->global_nonpersistent_units,
size HEAPACCT(which));
LOG(GLOBAL, LOG_HEAP, 6, "\nglobal nonpersistent alloc: " PFX " (%d bytes)\n", p,
size);
} else {
thread_units_t *nph = ((thread_heap_t *)dcontext->heap_field)->nonpersistent_heap;
p = common_heap_alloc(nph, size HEAPACCT(which));
}
ASSERT(p != NULL);
return p;
}
void
nonpersistent_heap_free(dcontext_t *dcontext, void *p,
size_t size HEAPACCT(which_heap_t which))
{
if (dcontext == GLOBAL_DCONTEXT) {
common_global_heap_free(&heapmgt->global_nonpersistent_units, p,
size HEAPACCT(which));
LOG(GLOBAL, LOG_HEAP, 6, "\nglobal nonpersistent free: " PFX " (%d bytes)\n", p,
size);
} else {
thread_units_t *nph = ((thread_heap_t *)dcontext->heap_field)->nonpersistent_heap;
DEBUG_DECLARE(bool ok =) common_heap_free(nph, p, size HEAPACCT(which));
ASSERT(ok);
}
}
void *
heap_reachable_alloc(dcontext_t *dcontext, size_t size HEAPACCT(which_heap_t which))
{
void *p;
* regular DR library and not just drdecode, to support libraries that would use
* drdecode but that also have to work with full DR (i#2499).
*/
if (heapmgt == &temp_heapmgt &&
* d_r_heap_exit() clears.
*/
!heapmgt->global_heap_writable) {
standalone_init();
}
if (!REACHABLE_HEAP()) {
if (dcontext == GLOBAL_DCONTEXT) {
p = common_global_heap_alloc(&heapmgt->global_reachable_units,
size HEAPACCT(which));
LOG(GLOBAL, LOG_HEAP, 6, "\nglobal reachable alloc: " PFX " (%d bytes)\n", p,
size);
} else {
thread_units_t *units =
((thread_heap_t *)dcontext->heap_field)->reachable_heap;
p = common_heap_alloc(units, size HEAPACCT(which));
}
} else {
p = heap_alloc(dcontext, size HEAPACCT(which));
}
ASSERT(p != NULL);
return p;
}
void
heap_reachable_free(dcontext_t *dcontext, void *p,
size_t size HEAPACCT(which_heap_t which))
{
if (!REACHABLE_HEAP()) {
if (dcontext == GLOBAL_DCONTEXT) {
common_global_heap_free(&heapmgt->global_reachable_units, p,
size HEAPACCT(which));
LOG(GLOBAL, LOG_HEAP, 6, "\nglobal reachable free: " PFX " (%d bytes)\n", p,
size);
} else {
thread_units_t *units =
((thread_heap_t *)dcontext->heap_field)->reachable_heap;
DEBUG_DECLARE(bool ok =) common_heap_free(units, p, size HEAPACCT(which));
ASSERT(ok);
}
} else {
heap_free(dcontext, p, size HEAPACCT(which));
}
}
* SPECIAL SINGLE-ALLOC-SIZE HEAP SERVICE
*/
* All allocations are of a single block size
* If use_lock is false, no synchronization is needed or even safe
*/
* 1) We don't always allocate the header inline.
* 2) We are sometimes executed from and so need pc prof support.
* 3) We don't need all the fields of heap_unit_t.
*/
typedef struct _special_heap_unit_t {
heap_pc alloc_pc;
heap_pc start_pc;
heap_pc end_pc;
heap_pc cur_pc;
heap_pc reserved_end_pc;
#ifdef WINDOWS_PC_SAMPLE
profile_t *profile;
#endif
#ifdef DEBUG
int id;
#endif
struct _special_heap_unit_t *next;
} special_heap_unit_t;
#define SPECIAL_UNIT_COMMIT_SIZE(u) ((u)->end_pc - (u)->alloc_pc)
#define SPECIAL_UNIT_RESERVED_SIZE(u) ((u)->reserved_end_pc - (u)->alloc_pc)
#define SPECIAL_UNIT_HEADER_INLINE(u) ((u)->alloc_pc != (u)->start_pc)
#define SPECIAL_UNIT_ALLOC_SIZE(u) (SPECIAL_UNIT_RESERVED_SIZE(u))
typedef struct _cfree_header {
struct _cfree_header *next_cfree;
uint count;
} cfree_header_t;
typedef struct _special_units_t {
special_heap_unit_t *top_unit;
special_heap_unit_t *cur_unit;
uint block_size;
uint block_alignment;
heap_pc free_list;
cfree_header_t *cfree_list;
#ifdef DEBUG
int num_units;
#endif
bool writable : 1;
bool executable : 1;
* (this isn't just an optimization, it's for correctness)
*/
bool use_lock : 1;
bool in_iterator : 1;
bool persistent : 1;
bool per_thread : 1;
mutex_t lock;
* so it's via a passed-in vector and passed-in data
*/
vm_area_vector_t *heap_areas;
void *lookup_retval;
#ifdef WINDOWS_PC_SAMPLE
struct _special_units_t *next;
#endif
#ifdef HEAP_ACCOUNTING
heap_acct_t acct;
#endif
} special_units_t;
#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
DECLARE_CXTSWPROT_VAR(static mutex_t special_units_list_lock,
INIT_LOCK_FREE(special_units_list_lock));
DECLARE_CXTSWPROT_VAR(static special_units_t *special_units_list, NULL);
#endif
#if defined(DEBUG) && defined(HEAP_ACCOUNTING) && defined(HOT_PATCHING_INTERFACE)
* the heap module in the same file. Part of fix for case 9593 that required
* leaking memory.
*/
static int
get_special_heap_header_size(void)
{
return sizeof(special_units_t);
}
#endif
#ifdef WINDOWS_PC_SAMPLE
static inline bool
special_heap_profile_enabled()
{
return (dynamo_options.profile_pcs && dynamo_options.prof_pcs_stubs >= 2 &&
dynamo_options.prof_pcs_stubs <= 32);
}
#endif
static inline uint
get_prot(special_units_t *su)
{
return (su->executable ? MEMPROT_READ | MEMPROT_WRITE | MEMPROT_EXEC
: MEMPROT_READ | MEMPROT_WRITE);
}
static inline which_vmm_t
get_which(special_units_t *su)
{
which_vmm_t which = VMM_SPECIAL_HEAP;
if (su->executable)
which |= VMM_REACHABLE;
if (su->per_thread)
which |= VMM_PER_THREAD;
return which;
}
static inline byte *
special_heap_get_writable_addr(special_units_t *su, byte *addr)
{
if (su->executable)
return vmcode_get_writable_addr(addr);
return addr;
}
static inline byte *
special_heap_get_executable_addr(special_units_t *su, byte *addr)
{
if (su->executable)
return vmcode_get_executable_addr(addr);
return addr;
}
static void
special_unit_extend_commitment(special_units_t *su, special_heap_unit_t *u,
size_t size_need, uint prot)
{
u->end_pc += common_heap_extend_commitment(u->cur_pc, u->end_pc, u->reserved_end_pc,
size_need, prot, get_which(su));
}
* If pc is non-NULL, allocates separate memory for the header, and
* uses pc for the heap region (assuming size is fully committed).
* unit_full only applies to the non-NULL case, indicating whether
* to continue to allocate from this unit.
*/
static special_heap_unit_t *
special_heap_create_unit(special_units_t *su, byte *pc, size_t size, bool unit_full)
{
special_heap_unit_t *u;
size_t commit_size;
uint prot = get_prot(su);
ASSERT_OWN_MUTEX(su->use_lock, &su->lock);
if (pc != NULL) {
u = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, special_heap_unit_t, ACCT_MEM_MGT,
PROTECTED);
ASSERT(u != NULL);
u->start_pc = pc;
u->alloc_pc = pc;
commit_size = size;
ASSERT(su->block_alignment == 0 || ALIGNED(u->start_pc, su->block_alignment));
} else {
ASSERT(ALIGNED(size, PAGE_SIZE));
commit_size = DYNAMO_OPTION(heap_commit_increment);
if (commit_size > size)
commit_size = size;
* linux), etc. will be acquired, and presumably !su->use_lock means
* user can't handle ANY lock being acquired, we warn here: xref PR
* 596768. In release build, we try to acqure the memory anyway. I'm
* worried about pcprofile: can only fit ~1K in one unit and so will
* easily run out...should it allocate additional units up front?
* => PR 596808.
*/
DODEBUG({
if (su->top_unit != NULL && !su->use_lock) {
SYSLOG_INTERNAL_WARNING_ONCE("potentially unsafe: allocating a new "
"fragile special heap unit!");
}
});
u = (special_heap_unit_t *)get_guarded_real_memory(
size, commit_size, prot, true, true, NULL,
get_which(su) _IF_DEBUG("special_heap"));
ASSERT(u != NULL);
* is a much narrower interface for executable addresses: just returning
* pointers on alloc, while we have many write points.
*/
u = (special_heap_unit_t *)special_heap_get_writable_addr(su, (byte *)u);
u->alloc_pc = (heap_pc)u;
u->start_pc = (heap_pc)(((ptr_uint_t)u) + sizeof(special_heap_unit_t));
if (su->block_alignment != 0) {
STATS_ADD(heap_special_align,
ALIGN_FORWARD(u->start_pc, su->block_alignment) -
(ptr_uint_t)u->start_pc);
u->start_pc = (heap_pc)ALIGN_FORWARD(u->start_pc, su->block_alignment);
}
}
u->end_pc = u->alloc_pc + commit_size;
u->reserved_end_pc = u->alloc_pc + size;
if (pc != NULL && unit_full) {
ASSERT(u->reserved_end_pc == u->end_pc);
u->cur_pc = u->end_pc;
} else
u->cur_pc = u->start_pc;
u->next = NULL;
DODEBUG({
u->id = su->num_units;
su->num_units++;
});
#ifdef WINDOWS_PC_SAMPLE
if (special_heap_profile_enabled()) {
u->profile = create_profile((app_pc)PAGE_START(u->start_pc), u->reserved_end_pc,
dynamo_options.prof_pcs_stubs, NULL);
start_profile(u->profile);
} else
u->profile = NULL;
#endif
* if !su->use_lock!
*/
RSTATS_ADD_PEAK(heap_capacity, commit_size);
RSTATS_ADD_PEAK(heap_special_capacity, commit_size);
STATS_ADD_PEAK(heap_special_units, 1);
STATS_ADD_PEAK(heap_reserved_only, (u->reserved_end_pc - u->end_pc));
if (su->heap_areas != NULL) {
vmvector_add(su->heap_areas, u->alloc_pc, u->reserved_end_pc, su->lookup_retval);
}
#ifdef DEBUG_MEMORY
DOCHECK(CHKLVL_MEMFILL, {
if (pc == NULL) {
memset(u->start_pc, HEAP_UNALLOCATED_BYTE, u->end_pc - u->start_pc);
}
});
#endif
return u;
}
static void *
special_heap_init_internal(uint block_size, uint block_alignment, bool use_lock,
bool executable, bool persistent, vm_area_vector_t *vector,
void *vector_data, byte *heap_region, size_t heap_size,
bool unit_full)
{
special_units_t *su;
size_t unit_size = heap_size;
if (block_alignment != 0)
block_size = ALIGN_FORWARD(block_size, block_alignment);
if (unit_size == 0) {
* pending queue) don't need a lot of space, so we have a smaller min size
* than regular heap units which use HEAP_UNIT_MIN_SIZE.
*/
unit_size = (block_size * 16 > PAGE_SIZE) ? (block_size * 16) : PAGE_SIZE;
}
if (heap_region == NULL) {
unit_size = (size_t)ALIGN_FORWARD(unit_size, PAGE_SIZE);
}
su = (special_units_t *)(persistent
? global_heap_alloc(sizeof(special_units_t)
HEAPACCT(ACCT_MEM_MGT))
: nonpersistent_heap_alloc(GLOBAL_DCONTEXT,
sizeof(special_units_t)
HEAPACCT(ACCT_MEM_MGT)));
memset(su, 0, sizeof(*su));
ASSERT(block_size >= sizeof(heap_pc *) && "need room for free list ptrs");
ASSERT(block_size >= sizeof(heap_pc *) + sizeof(uint) &&
"need room for cfree list ptrs");
su->block_size = block_size;
su->block_alignment = block_alignment;
su->executable = executable;
su->persistent = persistent;
su->per_thread = !use_lock;
su->writable = true;
su->free_list = NULL;
su->cfree_list = NULL;
DODEBUG({ su->num_units = 0; });
ASSERT((vector == NULL) == (vector_data == NULL));
su->heap_areas = vector;
su->lookup_retval = vector_data;
su->in_iterator = false;
if (use_lock)
ASSIGN_INIT_LOCK_FREE(su->lock, special_heap_lock);
* cannot acquire special_heap_lock -- so we do not acquire
* for the initial unit creation, which is safe since su is still
* private to this routine.
*/
su->use_lock = false;
su->top_unit = special_heap_create_unit(su, heap_region, unit_size, unit_full);
su->use_lock = use_lock;
#ifdef HEAP_ACCOUNTING
memset(&su->acct, 0, sizeof(su->acct));
#endif
su->cur_unit = su->top_unit;
#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
if (special_heap_profile_enabled()) {
d_r_mutex_lock(&special_units_list_lock);
su->next = special_units_list;
special_units_list = su;
d_r_mutex_unlock(&special_units_list_lock);
}
#endif
return su;
}
void *
special_heap_init(uint block_size, bool use_lock, bool executable, bool persistent)
{
uint alignment = 0;
* XXX: find those users and have them call special_heap_init_aligned()
* and removed this.
*/
if (IS_POWER_OF_2(block_size))
alignment = block_size;
return special_heap_init_internal(block_size, alignment, use_lock, executable,
persistent, NULL, NULL, NULL, 0, false);
}
void *
special_heap_init_aligned(uint block_size, uint alignment, bool use_lock, bool executable,
bool persistent, size_t initial_unit_size)
{
return special_heap_init_internal(block_size, alignment, use_lock, executable,
persistent, NULL, NULL, NULL, initial_unit_size,
false);
}
* (heap_region, heap_region+heap_size) whose fullness is unit_full. */
void *
special_heap_pclookup_init(uint block_size, bool use_lock, bool executable,
bool persistent, vm_area_vector_t *vector, void *vector_data,
byte *heap_region, size_t heap_size, bool unit_full)
{
uint alignment = 0;
if (IS_POWER_OF_2(block_size))
alignment = block_size;
return special_heap_init_internal(block_size, alignment, use_lock, executable,
persistent, vector, vector_data, heap_region,
heap_size, unit_full);
}
void
special_heap_set_vector_data(void *special, void *vector_data)
{
special_units_t *su = (special_units_t *)special;
special_heap_unit_t *u;
ASSERT(su->heap_areas != NULL);
* to set the data: we'd need to expose the iterator index or
* the vmarea struct rather than than the clean copy we have now
*/
for (u = su->top_unit; u != NULL; u = u->next) {
vmvector_modify_data(su->heap_areas, u->alloc_pc, u->reserved_end_pc,
vector_data);
}
}
* non-externally-allocated unit.
* Sets the cur pc for the only unit to end_pc.
*/
bool
special_heap_set_unit_end(void *special, byte *end_pc)
{
special_units_t *su = (special_units_t *)special;
if (su->top_unit->next != NULL || SPECIAL_UNIT_HEADER_INLINE(su->top_unit) ||
end_pc < su->top_unit->start_pc || end_pc > su->top_unit->end_pc)
return false;
su->top_unit->cur_pc = end_pc;
return true;
}
#ifdef WINDOWS_PC_SAMPLE
static void
special_heap_profile_stop(special_heap_unit_t *u)
{
int sum;
ASSERT(special_heap_profile_enabled());
stop_profile(u->profile);
sum = sum_profile(u->profile);
if (sum > 0) {
d_r_mutex_lock(&profile_dump_lock);
print_file(profile_file, "\nDumping special heap unit profile\n%d hits\n", sum);
dump_profile(profile_file, u->profile);
d_r_mutex_unlock(&profile_dump_lock);
}
}
#endif
#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
void
special_heap_profile_exit()
{
special_heap_unit_t *u;
special_units_t *su;
ASSERT(special_heap_profile_enabled());
d_r_mutex_lock(&special_units_list_lock);
for (su = special_units_list; su != NULL; su = su->next) {
if (su->use_lock)
d_r_mutex_lock(&su->lock);
for (u = su->top_unit; u != NULL; u = u->next) {
if (u->profile != NULL)
special_heap_profile_stop(u);
}
if (su->use_lock)
d_r_mutex_unlock(&su->lock);
}
d_r_mutex_unlock(&special_units_list_lock);
}
#endif
void
special_heap_exit(void *special)
{
special_units_t *su = (special_units_t *)special;
special_heap_unit_t *u, *next_u;
#ifdef DEBUG
size_t total_heap_used = 0;
#endif
u = su->top_unit;
while (u != NULL) {
DOLOG(1, LOG_HEAP | LOG_STATS, {
size_t num_used = u->cur_pc - u->start_pc;
total_heap_used += num_used;
LOG(THREAD_GET, LOG_HEAP | LOG_STATS, 1,
"Heap unit " SZFMT " (size " SZFMT " [/" SZFMT "] KB): used " SZFMT
" KB\n",
u->id, (SPECIAL_UNIT_COMMIT_SIZE(u)) / 1024,
SPECIAL_UNIT_RESERVED_SIZE(u) / 1024, num_used / 1024);
});
next_u = u->next;
#ifdef WINDOWS_PC_SAMPLE
if (u->profile != NULL) {
ASSERT(special_heap_profile_enabled());
special_heap_profile_stop(u);
free_profile(u->profile);
u->profile = NULL;
}
#endif
STATS_ADD(heap_special_units, -1);
RSTATS_SUB(heap_special_capacity, SPECIAL_UNIT_COMMIT_SIZE(u));
if (su->heap_areas != NULL) {
vmvector_remove(su->heap_areas, u->alloc_pc, u->reserved_end_pc);
}
if (!SPECIAL_UNIT_HEADER_INLINE(u)) {
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, u, special_heap_unit_t, ACCT_MEM_MGT,
PROTECTED);
} else {
release_guarded_real_memory((vm_addr_t)u, SPECIAL_UNIT_RESERVED_SIZE(u),
true , true,
get_which(su));
}
u = next_u;
}
#ifdef HEAP_ACCOUNTING
add_heapacct_to_global_stats(&su->acct);
#endif
LOG(THREAD_GET, LOG_HEAP | LOG_STATS, 1, "\tTotal heap used: " SZFMT " KB\n",
total_heap_used / 1024);
#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
if (special_heap_profile_enabled()) {
d_r_mutex_lock(&special_units_list_lock);
if (special_units_list == su)
special_units_list = su->next;
else {
special_units_t *prev = special_units_list;
ASSERT(prev != NULL);
for (; prev->next != NULL && prev->next != su; prev = prev->next)
;
ASSERT(prev->next == su);
prev->next = su->next;
}
d_r_mutex_unlock(&special_units_list_lock);
}
#endif
if (su->use_lock)
DELETE_LOCK(su->lock);
if (su->persistent) {
global_heap_free(su, sizeof(special_units_t) HEAPACCT(ACCT_MEM_MGT));
} else {
nonpersistent_heap_free(GLOBAL_DCONTEXT, su,
sizeof(special_units_t) HEAPACCT(ACCT_MEM_MGT));
}
}
void *
special_heap_calloc(void *special, uint num)
{
#ifdef DEBUG
dcontext_t *dcontext = get_thread_private_dcontext();
#endif
special_units_t *su = (special_units_t *)special;
special_heap_unit_t *u;
void *p = NULL;
bool took_free = false;
ASSERT(num > 0);
if (su->use_lock)
d_r_mutex_lock(&su->lock);
u = su->cur_unit;
if (su->free_list != NULL && num == 1) {
p = (void *)su->free_list;
su->free_list = *((heap_pc *)p);
took_free = true;
} else if (su->cfree_list != NULL && num > 1) {
* seems better to save the bigger pieces
*/
cfree_header_t *cfree = su->cfree_list, *prev = NULL;
while (cfree != NULL && cfree->count < num) {
prev = cfree;
cfree = cfree->next_cfree;
}
if (cfree != NULL) {
ASSERT(cfree->count >= num);
took_free = true;
if (cfree->count == num) {
if (prev == NULL)
su->cfree_list = cfree->next_cfree;
else
prev->next_cfree = cfree->next_cfree;
p = (void *)cfree;
} else if (cfree->count == num + 1) {
heap_pc tail = ((heap_pc)cfree) + num * su->block_size;
*((heap_pc *)tail) = su->free_list;
su->free_list = tail;
p = (void *)cfree;
} else {
cfree->count -= num;
p = (void *)(((heap_pc)cfree) + (cfree->count - num) * su->block_size);
}
}
}
if (!took_free) {
if (u->cur_pc + su->block_size * num > u->end_pc ||
POINTER_OVERFLOW_ON_ADD(u->cur_pc, su->block_size * num)) {
size_t pre_commit_size = SPECIAL_UNIT_COMMIT_SIZE(u);
special_unit_extend_commitment(su, u, su->block_size * num, get_prot(su));
RSTATS_ADD_PEAK(heap_special_capacity,
SPECIAL_UNIT_COMMIT_SIZE(u) - pre_commit_size);
if (u->cur_pc + su->block_size * num > u->end_pc ||
POINTER_OVERFLOW_ON_ADD(u->cur_pc, su->block_size * num)) {
special_heap_unit_t *new_unit;
special_heap_unit_t *prev = su->top_unit;
size_t size = SPECIAL_UNIT_ALLOC_SIZE(u);
ASSERT(ALIGNED(size, PAGE_SIZE));
while (prev->next != NULL)
prev = prev->next;
if (size * 2 <= HEAP_UNIT_MAX_SIZE)
size *= 2;
ASSERT(su->block_size * num < size);
new_unit = special_heap_create_unit(su, NULL, size, false );
prev->next = new_unit;
if (su->use_lock) {
LOG(THREAD, LOG_HEAP, 3,
"%s: Creating new heap unit %d " PFX "-" PFX "-" PFX "\n",
__FUNCTION__, new_unit->id, new_unit->alloc_pc, new_unit->end_pc,
new_unit->reserved_end_pc);
}
su->cur_unit = new_unit;
u = new_unit;
ASSERT(u->cur_pc + su->block_size * num <= u->end_pc &&
!POINTER_OVERFLOW_ON_ADD(u->cur_pc, su->block_size * num));
}
}
p = (void *)u->cur_pc;
u->cur_pc += su->block_size * num;
ACCOUNT_FOR_ALLOC(alloc_new, su, ACCT_SPECIAL, su->block_size * num,
su->block_size * num);
} else {
ACCOUNT_FOR_ALLOC(alloc_reuse, su, ACCT_SPECIAL, su->block_size * num,
su->block_size * num);
}
if (su->use_lock)
d_r_mutex_unlock(&su->lock);
#ifdef DEBUG_MEMORY
DOCHECK(CHKLVL_MEMFILL, memset(p, HEAP_ALLOCATED_BYTE, su->block_size * num););
#endif
ASSERT(p != NULL);
return (void *)special_heap_get_executable_addr(su, p);
}
void *
special_heap_alloc(void *special)
{
return special_heap_calloc(special, 1);
}
void
special_heap_cfree(void *special, void *p, uint num)
{
special_units_t *su = (special_units_t *)special;
ASSERT(num > 0);
ASSERT(p != NULL);
ASSERT(!su->in_iterator || OWN_MUTEX(&su->lock));
if (su->use_lock && !su->in_iterator)
d_r_mutex_lock(&su->lock);
p = (void *)special_heap_get_writable_addr(su, p);
#ifdef DEBUG_MEMORY
DOCHECK(CHKLVL_MEMFILL, memset(p, HEAP_UNALLOCATED_BYTE, su->block_size * num););
#endif
if (num == 1) {
*((heap_pc *)p) = su->free_list;
su->free_list = (heap_pc)p;
} else {
cfree_header_t *cfree = (cfree_header_t *)p;
cfree->next_cfree = su->cfree_list;
cfree->count = num;
su->cfree_list = (cfree_header_t *)p;
}
#ifdef HEAP_ACCOUNTING
ACCOUNT_FOR_FREE(su, ACCT_SPECIAL, su->block_size * num);
#endif
if (su->use_lock && !su->in_iterator)
d_r_mutex_unlock(&su->lock);
}
void
special_heap_free(void *special, void *p)
{
special_heap_cfree(special, p, 1);
}
bool
special_heap_can_calloc(void *special, uint num)
{
special_units_t *su = (special_units_t *)special;
bool can_calloc = false;
ASSERT(num > 0);
if (su->use_lock)
d_r_mutex_lock(&su->lock);
if (su->free_list != NULL && num == 1) {
can_calloc = true;
} else if (su->cfree_list != NULL && num > 1) {
cfree_header_t *cfree = su->cfree_list;
while (cfree != NULL) {
if (cfree->count >= num) {
can_calloc = true;
break;
}
cfree = cfree->next_cfree;
}
}
if (!can_calloc) {
special_heap_unit_t *u = su->cur_unit;
can_calloc = (u->cur_pc + su->block_size * num <= u->reserved_end_pc &&
!POINTER_OVERFLOW_ON_ADD(u->cur_pc, su->block_size * num));
}
if (su->use_lock)
d_r_mutex_unlock(&su->lock);
return can_calloc;
}
* grabs the heap lock (regardless of whether synch is used for allocs), and
* destroyed with special_heap_iterator_stop() to release the lock.
* If the special heap uses no lock for alloc, it is up to the caller
* to prevent race conditions causing problems.
* Accessor special_heap_iterator_next() should be called only when
* predicate special_heap_iterator_hasnext() is true.
* Any mutation of the heap while iterating will result in a deadlock
* for heaps that use locks for alloc, except for individual freeing,
* which will proceed w/o trying to grab the lock a second time.
* FIXME: could generalize to regular heaps if a use arises.
*/
void
special_heap_iterator_start(void *heap, special_heap_iterator_t *shi)
{
special_units_t *su = (special_units_t *)heap;
ASSERT(heap != NULL);
ASSERT(shi != NULL);
d_r_mutex_lock(&su->lock);
shi->heap = heap;
shi->next_unit = (void *)su->top_unit;
su->in_iterator = true;
}
bool
special_heap_iterator_hasnext(special_heap_iterator_t *shi)
{
ASSERT(shi != NULL);
DOCHECK(1, {
special_units_t *su = (special_units_t *)shi->heap;
ASSERT(su != NULL);
ASSERT_OWN_MUTEX(true, &su->lock);
});
return (shi->next_unit != NULL);
}
* Has to be initialized with special_heap_iterator_start, and should be
* called only when special_heap_iterator_hasnext() is true.
* Sets the area boundaries in area_start and area_end.
*/
void
special_heap_iterator_next(special_heap_iterator_t *shi ,
app_pc *heap_start , app_pc *heap_end )
{
special_units_t *su;
special_heap_unit_t *u;
ASSERT(shi != NULL);
su = (special_units_t *)shi->heap;
ASSERT(su != NULL);
ASSERT_OWN_MUTEX(true, &su->lock);
u = (special_heap_unit_t *)shi->next_unit;
ASSERT(u != NULL);
if (u != NULL) {
if (heap_start != NULL)
*heap_start = special_heap_get_executable_addr(su, u->start_pc);
ASSERT(u->cur_pc <= u->end_pc);
if (heap_end != NULL)
*heap_end = special_heap_get_executable_addr(su, u->cur_pc);
shi->next_unit = (void *)u->next;
}
}
void
special_heap_iterator_stop(special_heap_iterator_t *shi)
{
special_units_t *su;
ASSERT(shi != NULL);
su = (special_units_t *)shi->heap;
ASSERT(su != NULL);
ASSERT_OWN_MUTEX(true, &su->lock);
su->in_iterator = false;
d_r_mutex_unlock(&su->lock);
DODEBUG({
shi->heap = NULL;
shi->next_unit = NULL;
});
}
#if defined(DEBUG) && defined(HOT_PATCHING_INTERFACE)
* we can't delete the trampoline heap. However if that heap's lock isn't
* deleted, we'll assert. This routine is used only for that. Normally, we
* should call special_heap_exit() which deletes the lock. */
void
special_heap_delete_lock(void *special)
{
special_units_t *su = (special_units_t *)special;
ASSERT(su != NULL);
if (su == NULL)
return;
ASSERT(su->use_lock);
if (su->use_lock)
DELETE_LOCK(su->lock);
}
#endif
#ifdef WINDOWS
* landing pad areas, each of which contains multiple landing pads. Landing
* pads are small trampolines used to jump from the hook point to the main
* trampoline. This is used in both 32-bit and 64-bit DR. In both cases it
* will handle the problem of hook chaining by 3rd party software and us having
* to release our hooks (we'll nop the landing pad and free the trampoline).
* In 64-bit it also solves the problem of reachability of the 5-byte rel jmp
* we use for hooking, i.e., that 5-byte rel jmp may not reach the main
* trampoline in DR heap. We have to maintain the hook a 5-byte jmp because
* hotp_only assumes it (see PR 250294).
*
* A landing pad will have nothing more than a jump (5-byte rel for 32-bit DR
* and 64-bit abs ind jmp for 64-bit DR) to the trampoline and a 5-byte rel jmp
* back to the next instruction after the hook, plus the displaced app instrs.
*
* To handle hook chaining landing pads won't be released till process exit
* (not on a detach), their first jump will just be nop'ed. As landing pads
* aren't released till exit, all landing pads are just incrementally allocated
* in a landing pad area.
*
* Note: Landing pad areas don't necessarily have to fall within the vm_reserve
* region or capacity, so aren't accounted by our vmm.
*
* Note: If in future other needs for such region specific allocation should
* arise, then we should convert this into special_heap_alloc_in_region(). For
* now, landing pads are the only consumers, so was decided to be acceptable.
*
* See win32/callback.c for emit_landing_pad_code() and landing pad usage.
*/
typedef struct {
byte *start;
byte *end;
byte *commit_end;
byte *cur_ptr;
bool allocated;
} landing_pad_area_t;
* its trampoline via the landing pad. The landing pad will reachable by a
* 32-bit relative jmp from addr_to_hook.
* Note: we may want to generalize this at some point such that the size of the
* landing pad is passed as an argument.
*
* For Windows we assume that landing_pads_to_executable_areas(true) will be
* called once landing pads are finished being created.
*/
byte *
alloc_landing_pad(app_pc addr_to_hook)
{
app_pc hook_region_start, hook_region_end;
app_pc alloc_region_start, alloc_region_end;
app_pc lpad_area_start = NULL, lpad_area_end;
app_pc lpad = NULL;
landing_pad_area_t *lpad_area = NULL;
* or memory region containing addr_to_hook can use the same area for a
* landing pad. Makes it more efficient. */
hook_region_start = get_allocation_base(addr_to_hook);
if (hook_region_start == NULL) {
ASSERT_CURIOSITY("trying to hook raw or unallocated memory?");
hook_region_start = addr_to_hook;
hook_region_end = addr_to_hook;
} else {
hook_region_end =
hook_region_start + get_allocation_size(hook_region_start, NULL);
ASSERT(hook_region_end > hook_region_start);
* images to 2 GB. Also, if region is > 2 GB the reachability macros
* called below will return a region smaller (and with start and end
* inverted) than the region from which the reachability is desired,
* i.e., some of the areas in [hook_region_start, hook_region_end)
* won't be able to reach the region computed.
*
* A better choice is to pick something smaller (100 MB) because if the
* region is close to 2 GB in size then we might not be able to
* allocate memory for a landing pad that is reachable.
*/
if (hook_region_end - hook_region_start > 100 * 1024 * 1024) {
ASSERT_CURIOSITY(false && "seeing patch region > 100 MB - DGC?");
hook_region_start = MIN(
addr_to_hook, MAX(hook_region_start, addr_to_hook - 50 * 1024 * 1024));
hook_region_end =
MAX(addr_to_hook, MIN(hook_region_end, addr_to_hook + 50 * 1024 * 1024));
}
}
* hook region with a 32-bit rel jmp.
*/
alloc_region_start = REACHABLE_32BIT_START(hook_region_start, hook_region_end);
alloc_region_end = REACHABLE_32BIT_END(hook_region_start, hook_region_end);
ASSERT(alloc_region_start < alloc_region_end);
* region for the hook location. If so use it, else allocate one.
*/
d_r_write_lock(&landing_pad_areas->lock);
if (vmvector_overlap(landing_pad_areas, alloc_region_start, alloc_region_end)) {
* within alloc_region_start and alloc_region_end. If a landing pad
* area is only partially within the alloc region, then a landing pad
* created there won't be able to reach the addr_to_hook. BTW, that
* landing pad area should have enough space to allocate a landing pad!
* If these conditions are met allocate a landing pad.
*/
vmvector_iterator_t lpad_area_iter;
vmvector_iterator_start(landing_pad_areas, &lpad_area_iter);
while (vmvector_iterator_hasnext(&lpad_area_iter)) {
lpad_area =
vmvector_iterator_next(&lpad_area_iter, &lpad_area_start, &lpad_area_end);
if (lpad_area_start < alloc_region_end &&
lpad_area_end > alloc_region_start &&
(lpad_area->cur_ptr + LANDING_PAD_SIZE) < lpad_area_end) {
* committed, if not commit more memory.
*/
if ((lpad_area->cur_ptr + LANDING_PAD_SIZE) >= lpad_area->commit_end) {
ASSERT(lpad_area->allocated);
extend_commitment(lpad_area->commit_end, PAGE_SIZE,
MEMPROT_READ | MEMPROT_EXEC,
false , VMM_SPECIAL_MMAP);
lpad_area->commit_end += PAGE_SIZE;
}
* allocate the landing pad.
*/
lpad = lpad_area->cur_ptr;
lpad_area->cur_ptr += LANDING_PAD_SIZE;
break;
}
}
vmvector_iterator_stop(&lpad_area_iter);
}
* allocation region or none fully contained within the allocation region,
* then create a new one within the allocation region. Then allocate a
* landing pad in it.
*/
if (lpad == NULL) {
bool allocated = true;
heap_error_code_t heap_error;
lpad_area_end = NULL;
lpad_area_start = os_heap_reserve_in_region(
(void *)ALIGN_FORWARD(alloc_region_start, PAGE_SIZE),
(void *)ALIGN_BACKWARD(alloc_region_end, PAGE_SIZE), LANDING_PAD_AREA_SIZE,
&heap_error, true );
if (lpad_area_start == NULL || heap_error == HEAP_ERROR_CANT_RESERVE_IN_REGION) {
* have made the region so large that there's nothing nearby to
* reserve.
*/
lpad_area_start = os_heap_reserve(
(void *)ALIGN_FORWARD(addr_to_hook, LANDING_PAD_AREA_SIZE),
LANDING_PAD_AREA_SIZE, &heap_error, true );
# ifdef WINDOWS
if (lpad_area_start == NULL &&
* XXX: support multiple "allocs" inside libs.
*/
vmvector_empty(landing_pad_areas) &&
os_find_free_code_space_in_libs(&lpad_area_start, &lpad_area_end)) {
if (lpad_area_end - lpad_area_start >= LANDING_PAD_SIZE &&
make_hookable(lpad_area_start, lpad_area_end - lpad_area_start,
NULL)) {
allocated = false;
* called once landing pads are finished being created and we
* can restore to +rx there.
*/
lpad_temp_writable_start = lpad_area_start;
lpad_temp_writable_size = lpad_area_end - lpad_area_start;
} else
lpad_area_start = NULL;
}
# endif
if (lpad_area_start == NULL) {
* theoretically 2 GB of dlls might get packed together before
* we get control (very unlikely), so we can fail. If it does,
* then say 'oom' and exit.
*/
SYSLOG_INTERNAL_WARNING("unable to reserve memory for landing pads");
report_low_on_memory(VMM_SPECIAL_MMAP | VMM_REACHABLE, OOM_RESERVE,
heap_error);
}
}
* and add it to landing_pad_areas vector. Note, we only commit 4k
* initially even though we reserve 64k (LANDING_PAD_AREA_SIZE), to
* avoid wastage.
*/
if (allocated) {
extend_commitment(lpad_area_start, PAGE_SIZE, MEMPROT_READ | MEMPROT_EXEC,
true , VMM_SPECIAL_MMAP);
}
lpad_area =
HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, landing_pad_area_t, ACCT_VMAREAS, PROTECTED);
lpad_area->start = lpad_area_start;
lpad_area->end = (lpad_area_end == NULL ? lpad_area_start + LANDING_PAD_AREA_SIZE
: lpad_area_end);
lpad_area->commit_end = lpad_area_start + PAGE_SIZE;
lpad_area->cur_ptr = lpad_area_start;
lpad_area->allocated = allocated;
lpad = lpad_area->cur_ptr;
lpad_area->cur_ptr += LANDING_PAD_SIZE;
vmvector_add(landing_pad_areas, lpad_area->start, lpad_area->end, lpad_area);
STATS_INC(num_landing_pad_areas);
}
* landing pads should be added. Only the ones used for DR hooks should be
* added to executable areas (which is done using
* landing_pads_to_executable_areas() at the end of inserting DR hooks).
* hotp_only related landing pads shouldn't be added to executable areas as
* their trampolines aren't added to executable_areas. This is why landing
* pads aren't added to executable_areas here, at the point of allocation.
*/
LOG(GLOBAL, LOG_ALL, 3, "%s: used " PIFX " bytes in " PFX "-" PFX "\n", __FUNCTION__,
lpad_area->cur_ptr - lpad_area->start, lpad_area->start, lpad_area->end);
ASSERT(lpad_area->cur_ptr <= lpad_area->end);
d_r_write_unlock(&landing_pad_areas->lock);
return lpad;
}
* recently allocated landing pad to the actual space used.
* Will fail if another landing pad was allocated between lpad_start
* being allocated and this routine being called.
*/
bool
trim_landing_pad(byte *lpad_start, size_t space_used)
{
landing_pad_area_t *lpad_area = NULL;
bool res = false;
d_r_write_lock(&landing_pad_areas->lock);
if (vmvector_lookup_data(landing_pad_areas, lpad_start, NULL, NULL, &lpad_area)) {
if (lpad_start == lpad_area->cur_ptr - LANDING_PAD_SIZE) {
lpad_area->cur_ptr -= (LANDING_PAD_SIZE - space_used);
res = true;
}
}
d_r_write_unlock(&landing_pad_areas->lock);
return res;
}
* landing pad areas. This is done to prevent bb building from considering
* landing pads to be selfmod code; as such, these don't have to be
* {add,remov}ed from executable_areas for hotp_only or for thin_client mode.
*/
void
landing_pads_to_executable_areas(bool add)
{
vmvector_iterator_t lpad_area_iter;
app_pc lpad_area_start, lpad_area_end;
DEBUG_DECLARE(landing_pad_area_t * lpad_area;)
uint lpad_area_size;
if (RUNNING_WITHOUT_CODE_CACHE())
return;
# ifdef WINDOWS
if (add && lpad_temp_writable_start != NULL) {
make_unhookable(lpad_temp_writable_start, lpad_temp_writable_size, true);
lpad_temp_writable_start = NULL;
}
# endif
* dr hooks in ntdll. For 64-bit, the image entry hook will result in a
* new landing pad.
*/
IF_X64_ELSE(, ASSERT(landing_pad_areas->length == 1);)
vmvector_iterator_start(landing_pad_areas, &lpad_area_iter);
while (vmvector_iterator_hasnext(&lpad_area_iter)) {
DEBUG_DECLARE(lpad_area =)
vmvector_iterator_next(&lpad_area_iter, &lpad_area_start, &lpad_area_end);
lpad_area_size = (uint)(lpad_area_end - lpad_area_start);
ASSERT(lpad_area_size <= LANDING_PAD_AREA_SIZE);
ASSERT(lpad_area->cur_ptr < lpad_area_end);
if (add) {
add_executable_region(lpad_area_start,
lpad_area_size _IF_DEBUG(
"add landing pad areas after inserting dr hooks"));
} else {
remove_executable_region(lpad_area_start, lpad_area_size,
false );
}
}
vmvector_iterator_stop(&lpad_area_iter);
}
* landing pad area. However, release all landing pads only on process exit;
* for detach leave the landing pads in (in case some one hooks after us they
* shouldn't crash if they chain correctly).
*/
static void
release_landing_pad_mem(void)
{
vmvector_iterator_t lpad_area_iter;
app_pc lpad_area_start, lpad_area_end;
landing_pad_area_t *lpad_area;
heap_error_code_t heap_error;
vmvector_iterator_start(landing_pad_areas, &lpad_area_iter);
while (vmvector_iterator_hasnext(&lpad_area_iter)) {
bool allocated;
lpad_area =
vmvector_iterator_next(&lpad_area_iter, &lpad_area_start, &lpad_area_end);
allocated = lpad_area->allocated;
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, lpad_area, landing_pad_area_t, ACCT_VMAREAS,
PROTECTED);
if (!doing_detach &&
allocated)
os_heap_free(lpad_area_start, LANDING_PAD_AREA_SIZE, &heap_error);
}
vmvector_iterator_stop(&lpad_area_iter);
vmvector_delete_vector(GLOBAL_DCONTEXT, landing_pad_areas);
}
#endif