* Copyright (c) 2011-2022 Google, Inc. All rights reserved.
* Copyright (c) 2000-2010 VMware, Inc. All rights reserved.
* **********************************************************/
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* * Neither the name of VMware, Inc. nor the names of its contributors may be
* used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
* DAMAGE.
*/
* fcache.c - fcache manager
*/
#include "globals.h"
#include "link.h"
#include "fragment.h"
#include "fcache.h"
#include "monitor.h"
#ifdef HOT_PATCHING_INTERFACE
# include "hotpatch.h"
#endif
#include <stddef.h>
#include <limits.h>
#include "perscache.h"
#include "synch.h"
#include "instrument.h"
* We grow a unit by resizing, shifting, and relinking, up to a maximum size,
* at which point we create a separate unit if we need more space
* Cache is extremely flexible in allowing resizing (hard to support)
* and separate units of different sizes, in any combination.
* We will build a unit larger than cache_{bb,trace}_unit_max for a single large request,
* up to the max cache size.
* To save memory, we don't make units larger than 64KB. Not much advantage
* to have huge units.
*/
#define FCACHE_OPTION(o) dynamo_options.o
* unit initial size is FCACHE_OPTION(cache_{bb,trace}_unit_init, default is 32*1024
* it grows by 4X steps up to FCACHE_OPTION(cache_{bb,trace}_unit_quadruple, default
* is 32*1024
* unit max size is FCACHE_OPTION(cache_{bb,trace}_unit_max, default is 64*1024
* once at max size, we make new units, all of max size
*
* thus default is to do no quadrupling, just a single doubling and then no more resizing
* FIXME: should we stop resizing altogether and just have variable-sized
* separate units? it's not like a 32K unit is too small to keep around...
* OTOH, we want the flexibility of resizing, for servers apps with lots of threads
* we may move the initial unit size smaller
*/
* fragment inside it in size (this can make it larger than cache_{bb,trace}_unit_max)
*/
#define MAX_SINGLE_MULTIPLE 2
* first expansion(s) free
* FCACHE_OPTION(cache_{bb,trace}_unit_upgrade, default is 64KB so 32=>64 is free
* after that, only expand if regenerated/replaced ratio matches these
* numbers (no floating-point, so we use 2 ints):
* dynamo_options.cache_{bb,trace}_regen default = 10
* dynamo_options.cache_{bb,trace}_replace default = 50
* special cases:
* if cache_{bb,trace}_regen == 0, never increases cache size after free upgrade
* if cache_{bb,trace}_replace == 0, always increases (effectively
* disabling adaptive working set, although the nice way to disable
* is to use -no_finite_{bb,trace}_cache)
*/
* Maximum cache sizes are stored in these two options:
* dynamo_options.cache_bb_max
* dynamo_options.cache_trace_max
* A value of 0 means "infinite"
*/
* for private cache and so will be filled even if it means bumping adjacent guys.
* For shared cache, we currently don't fill empty slots, but once we do we will
* only fill with a fragment that fits (case 4485). Shared empties are not prepended
* but rather are placed in the same location in the FIFO as the deleted guy.
* Should be the minimum common fragment size.
*/
#define MIN_EMPTY_HOLE(cache) \
MAX(((cache)->is_trace \
? 64U \
: ((!(cache)->is_shared && DYNAMO_OPTION(separate_private_stubs)) \
? 20U \
: (((cache)->is_shared && DYNAMO_OPTION(separate_shared_stubs)) \
? 20U \
: 64U))), \
MIN_FCACHE_SLOT_SIZE(cache))
* This is 2x the smallest fragment size
* FIXME: use larger size for trace cache?
*/
#define MIN_UNIT_END_HOLE(cache) ((uint)2 * MIN_EMPTY_HOLE(cache))
#define START_PC_ALIGNMENT 4
* to 4 to avoid sub-word fetches. NOTE we also need at least START_PC_ALIGNMENT
* byte alignment for the start_pc padding for -pad_jmps_shift_{bb,trace} support
* (need at least that much even without the option since we back align the
* start_pc to get the header)
*/
#define SLOT_ALIGNMENT(cache) \
((cache)->is_trace ? DYNAMO_OPTION(cache_trace_align) \
: ((cache)->is_coarse ? DYNAMO_OPTION(cache_coarse_align) \
: DYNAMO_OPTION(cache_bb_align)))
* *(fragment_t **) when working with the backpointer. If are to add
* more fields should use this. Although fragment_t may be a better
* place to keep such information. */
typedef struct _live_header_t {
fragment_t *f;
} live_header_t;
* start_pc is properly aligned (for -pad_jmps_shift_{bb,trace} support) */
#define HEADER_SIZE(f) (TEST(FRAG_COARSE_GRAIN, (f)->flags) ? 0 : (sizeof(fragment_t *)))
#define HEADER_SIZE_FROM_CACHE(cache) ((cache)->is_coarse ? 0 : (sizeof(fragment_t *)))
* We use a FIFO replacement strategy
* Empty slots in the cache are always at the front of the list, and they
* take the form of the empty_slot_t struct.
* Target fragments to delete are represented in the FIFO by their fragment_t
* struct, using the next_fcache field to chain them.
* We use a header for each fragment so we can delete adjacent-in-cache
* (different from indirected FIFO list) to make contiguous space available.
* Padding for alignment is always at the end of the fragment, so it can be
* combined with empty space eaten up when deleting a fragment but only needing
* some of its room. We assume everything starts out aligned, and the same alignment
* on the end of each slot keeps it aligned.
* Thus:
* ------
* header
* <up to START_PC_ALIGNMENT-1 bytes padding, stored in alignment of start_pc>
* start_pc: prefix
* body
* stubs
* <padding for alignment>
* ------
* header
* <up to START_PC_ALIGNMENT-1 bytes padding, stored in alignment of start_pc>
* start_pc: ...
*/
typedef struct _empty_slot_t {
cache_pc start_pc;
* we use flags==FRAG_IS_EMPTY_SLOT to indicate an empty slot
*/
uint flags;
fragment_t *next_fcache;
fragment_t *prev_fcache;
uint fcache_size;
* not just ushort so we can merge adjacents;
* not size_t since each unit assumed <4GB. */
} empty_slot_t;
#define FRAG_EMPTY(f) (TEST(FRAG_IS_EMPTY_SLOT, (f)->flags))
#define FRAG_START(f) \
(TEST(FRAG_IS_EMPTY_SLOT, (f)->flags) ? ((empty_slot_t *)(f))->start_pc \
: (f)->start_pc)
* once that fragment is in a cache, as contig-cache-walkers need a consistent view!
* FIXME: we can't assert as we can't do a unit lookup at all use sites
*/
#define FRAG_START_ASSIGN(f, val) \
do { \
if (TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) \
((empty_slot_t *)(f))->start_pc = (val); \
else \
(f)->start_pc = (val); \
} while (0)
* to START_PC_ALIGNMENT-1 bytes, back align to get the right header pointer */
#define FRAG_START_PADDING(f) \
((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags) || !PAD_JMPS_SHIFT_START((f)->flags)) \
? 0 \
: (ASSERT(CHECK_TRUNCATE_TYPE_uint( \
(ptr_uint_t)((f)->start_pc - \
ALIGN_BACKWARD((f)->start_pc, START_PC_ALIGNMENT)))), \
(uint)((ptr_uint_t)(f)->start_pc - \
ALIGN_BACKWARD((f)->start_pc, START_PC_ALIGNMENT))))
#define FRAG_HDR_START(f) (FRAG_START(f) - HEADER_SIZE(f) - FRAG_START_PADDING(f))
#define FRAG_SIZE(f) \
((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) \
? ((empty_slot_t *)(f))->fcache_size \
: (f)->size + (uint)(f)->fcache_extra + FRAG_START_PADDING(f))
* once that fragment is in a cache, as contig-cache-walkers need a consistent view!
* FIXME: we can't assert as we can't do a unit lookup at all use sites
*/
#define FRAG_SIZE_ASSIGN(f, val) \
do { \
if (TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) { \
ASSERT_TRUNCATE(((empty_slot_t *)(f))->fcache_size, uint, val); \
((empty_slot_t *)(f))->fcache_size = (val); \
} else { \
\
uint extra_tmp = ((val) - ((f)->size + FRAG_START_PADDING(f))); \
ASSERT_TRUNCATE((f)->fcache_extra, byte, extra_tmp); \
(f)->fcache_extra = (byte)extra_tmp; \
} \
} while (0)
#define FIFO_NEXT(f) \
((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) \
? ((empty_slot_t *)(f))->next_fcache \
: (ASSERT(!TEST(FRAG_SHARED, (f)->flags)), \
((private_fragment_t *)(f))->next_fcache))
#define FIFO_NEXT_ASSIGN(f, val) \
do { \
if (TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) \
((empty_slot_t *)(f))->next_fcache = (val); \
else { \
ASSERT(!TEST(FRAG_SHARED, (f)->flags)); \
((private_fragment_t *)(f))->next_fcache = (val); \
} \
} while (0)
#define FIFO_PREV(f) \
((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) \
? ((empty_slot_t *)(f))->prev_fcache \
: (ASSERT(!TEST(FRAG_SHARED, (f)->flags)), \
((private_fragment_t *)(f))->prev_fcache))
#define FIFO_PREV_ASSIGN(f, val) \
do { \
if (TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) \
((empty_slot_t *)(f))->prev_fcache = (val); \
else { \
ASSERT(!TEST(FRAG_SHARED, (f)->flags)); \
((private_fragment_t *)(f))->prev_fcache = (val); \
} \
} while (0)
#define FRAG_TAG(f) \
((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) ? ((empty_slot_t *)(f))->start_pc : (f)->tag)
#ifdef DEBUG
# define FRAG_ID(f) ((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) ? -1 : (f)->id)
#endif
#define FIFO_UNIT(f) \
(fcache_lookup_unit((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) \
? (((empty_slot_t *)(f))->start_pc) \
: ((f)->start_pc)))
* fragments. Instead they use a free list (below).
*/
#define USE_FIFO(f) (!TEST(FRAG_SHARED, (f)->flags))
#define USE_FREE_LIST(f) \
(TEST(FRAG_SHARED, (f)->flags) && !TEST(FRAG_COARSE_GRAIN, (f)->flags))
#define USE_FIFO_FOR_CACHE(c) (!(c)->is_shared)
#define USE_FREE_LIST_FOR_CACHE(c) ((c)->is_shared && !(c)->is_coarse)
* Free empty slot lists of different sizes for shared caches, where we
* cannot easily delete victims to make room in too-small slots.
*/
static const uint FREE_LIST_SIZES[] = {
* sizes, each bucket contains free slots that are in
* [SIZE[bucket], SIZE[bucket+1]) where the top one is infinite.
* case 7318 about further extensions
*
* Free slots in the current unit are added only if in the middle
* of the unit (the last ones are turned back into unclaimed space).
*
* Tuned for bbs: smallest are 40 (with stubs), plurality are 56, distribution trails
* off slowly beyond that. Note that b/c of pad_jmps we request more than
* the final sizes.
* FIXME: these #s are for inlined stubs, should re-tune w/ separate stubs
* (case 7163)
*/
0, 44, 52, 56, 64, 72, 80, 112, 172
};
#define FREE_LIST_SIZES_NUM (BUFFER_SIZE_ELEMENTS(FREE_LIST_SIZES))
* next-free and prev-free pointer and a size at the top of the empty slot, and
* don't waste memory with an empty_slot_t data structure.
* We also use flags field to allow distinguishing free list slots
* from live fragment_t's (see notes by flags field below).
*
* Our free list coalescing assumes that a fragment_t that follows a
* free list entry has the FRAG_FOLLOWS_FREE_ENTRY flag set.
*
* FIXME: could avoid heap w/ normal empty_slot_t scheme: if don't have
* separate stubs, empty_slot_t @ 20 bytes (no start_pc) should fit in
* any cache slot. Could save a few MB of heap on large apps.
* (This is case 4937.)
*
* FIXME: If free lists work well we could try using for private
* caches as well, instead of the empty_slot_t-struct-on-FIFO scheme.
*
* FIXME: unit pointer may be useful to avoid fcache_lookup_unit
*/
typedef struct _free_list_header_t {
struct _free_list_header_t *next;
* at the proper bit as though this were a "uint flags" at the same offset
* in the struct as fragment_t.flags. Since no one else examines a free list
* as though it might be a fragment_t, we don't care about the other flags.
* We have an ASSERT in fcache_init() to ensure the byte ordering is right.
*
* Since we compare a fragment_t* to this inlined struct, we're really
* comparing a fragment_t* to the first field, the next pointer. So when we
* de-reference the flags we're looking at the flags of the next entry in
* the free list. Thus to identify a free list entry we must check for
* either NULL or for the FRAG_FCACHE_FREE_LIST flag.
*/
uint flags;
* and can get larger. We thus make space for a larger size (i#4434), as the
* only downside is a smaller MIN_FCACHE_SLOT_SIZE, which is still small enough.
*/
uint size;
struct _free_list_header_t *prev;
} free_list_header_t;
* Combined w/ the FRAG_FOLLOWS_FREE_ENTRY flag this allows us to coalesce
* new free list entries with existing previous entries.
*/
typedef struct _free_list_footer_t {
uint size;
} free_list_footer_t;
#define MAX_FREE_ENTRY_SIZE UINT_MAX
* we're checking the next free list entry's flags by dereferencing, forcing a
* check for NULL as well (== end of list).
*/
#define FRAG_IS_FREE_LIST(f) ((f) == NULL || TEST(FRAG_FCACHE_FREE_LIST, (f)->flags))
* fcache slot, given a pc and a size for the current slot.
*/
#define FRAG_NEXT_SLOT(pc, size) (*((fragment_t **)((pc) + (size))))
#define FRAG_NEXT_FREE(pc, size) ((free_list_header_t *)((pc) + (size)))
* Current smallest bb is 5 bytes (single jmp) align-4 + header is 12,
* and we're at 20 here, so we are wasting some space, but few
* fragments are under 20: some are at 16 for 32-bit but almost none
* are smaller (see request_size_histogram[]).
*/
#define MIN_FCACHE_SLOT_SIZE(cache) \
((cache)->is_coarse ? 0 : (sizeof(free_list_header_t) + sizeof(free_list_footer_t)))
* single mmapped piece of cache
*/
struct _fcache;
#define UNIT_RESERVED_SIZE(u) ((size_t)((u)->reserved_end_pc - (u)->start_pc))
typedef struct _fcache_unit_t {
cache_pc start_pc;
cache_pc end_pc;
cache_pc cur_pc;
cache_pc reserved_end_pc;
size_t size;
bool full;
struct _fcache *cache;
#if defined(SIDELINE) || defined(WINDOWS_PC_SAMPLE)
dcontext_t *dcontext;
#endif
bool writable;
#ifdef WINDOWS_PC_SAMPLE
* field has been invalidated
*/
bool was_trace;
bool was_shared;
profile_t *profile;
#endif
bool per_thread;
bool pending_free;
#ifdef DEBUG
bool pending_flush;
#endif
uint flushtime;
* used only for units_to_free list, else 0 */
struct _fcache_unit_t *next_global;
struct _fcache_unit_t *prev_global;
struct _fcache_unit_t *next_local;
} fcache_unit_t;
#ifdef DEBUG
enum {
HISTOGRAM_GRANULARITY = 4,
HISTOGRAM_MAX_SIZE = 256,
};
#endif
* one "code cache" of a single type of fragment, made up of potentially
* multiple FcacheUnits
*/
typedef struct _fcache {
bool is_trace : 1;
bool is_shared : 1;
#ifdef DEBUG
* We only use this flag to get around lock ordering issues w/
* persistent caches and we don't bother to set it for all
* private caches.
*/
bool is_local : 1;
#endif
bool is_coarse : 1;
fragment_t *fifo;
* also includes empty slots as EmptySlots
* (all empty slots are at front of FIFO) */
fcache_unit_t *units;
* of the list is the only potentially
* non-full unit */
size_t size;
* and cleaner to have dedicated lock
*/
mutex_t lock;
#ifdef DEBUG
const char *name;
bool consistent;
#endif
coarse_info_t *coarse_info;
* type every time -- this also allows flexibility if we ever want
* to have different parameters per thread or something.
* not much of a space hit at all since there are 2 caches per thread
* and then 2 global caches.
*/
size_t max_size;
size_t max_unit_size;
size_t max_quadrupled_unit_size;
size_t free_upgrade_size;
size_t init_unit_size;
bool finite_cache;
uint regen_param;
uint replace_param;
uint num_regenerated;
uint num_replaced;
* too many checks when parameters are such that regen<<replace
*/
int wset_check;
* recording num_regenerated and num_replaced
*/
bool record_wset;
free_list_header_t *free_list[FREE_LIST_SIZES_NUM];
#ifdef DEBUG
uint free_stats_freed[FREE_LIST_SIZES_NUM];
uint free_stats_reused[FREE_LIST_SIZES_NUM];
uint free_stats_coalesced[FREE_LIST_SIZES_NUM];
uint free_stats_split[FREE_LIST_SIZES_NUM];
uint free_stats_charge[FREE_LIST_SIZES_NUM];
uint request_size_histogram[HISTOGRAM_MAX_SIZE / HISTOGRAM_GRANULARITY];
uint free_size_histogram[HISTOGRAM_MAX_SIZE / HISTOGRAM_GRANULARITY];
#endif
} fcache_t;
* per-thread structure
*/
typedef struct _fcache_thread_units_t {
fcache_t *bb;
fcache_t *trace;
cache_pc pending_unmap_pc;
size_t pending_unmap_size;
bool pending_flush;
} fcache_thread_units_t;
#define ALLOC_DC(dc, cache) ((cache)->is_shared ? GLOBAL_DCONTEXT : (dc))
* the lower-ranked shared_vm_areas lock, but the allsynch makes it safe to not
* acquire it.
*/
#define PROTECT_CACHE(cache, op) \
do { \
if ((cache)->is_shared && !is_self_allsynch_flushing()) { \
d_r_mutex_##op(&(cache)->lock); \
} \
} while (0);
#define CACHE_PROTECTED(cache) \
(!(cache)->is_shared || (cache)->is_local || OWN_MUTEX(&(cache)->lock) || \
dynamo_all_threads_synched)
* global, unique thread-shared structure:
*/
typedef struct _fcache_list_t {
fcache_unit_t *units;
fcache_unit_t *dead;
* for release build too, so it's separate...can we do better?
*/
uint num_dead;
* and kept on the live units list. Protected by unit_flush_lock, NOT by
* allunits_lock.
* We keep these list pointers on the heap for selfprot (case 8074).
*/
fcache_unit_t *units_to_flush;
* increasing flushtime, with a tail pointer to make appends easy
*/
fcache_unit_t *units_to_free;
fcache_unit_t *units_to_free_tail;
} fcache_list_t;
static fcache_list_t *allunits;
DECLARE_CXTSWPROT_VAR(static mutex_t allunits_lock, INIT_LOCK_FREE(allunits_lock));
DECLARE_CXTSWPROT_VAR(static mutex_t unit_flush_lock, INIT_LOCK_FREE(unit_flush_lock));
static fcache_t *shared_cache_bb;
static fcache_t *shared_cache_trace;
* we use an interval data structure rather than waste space with a
* backpointer in each fragment.
* Non-static so that synch routines in os.c can check its lock before calling
* is_pc_recreatable which calls in_fcache.
* Kept on the heap for selfprot (case 7957).
*/
vm_area_vector_t *fcache_unit_areas;
* all threads are suspended and so no synch is needed)
*/
DECLARE_FREQPROT_VAR(bool reset_in_progress, false);
* FIXME: use separate locks for separate triggers?
* reset_at_nth_thread is wholly inside dynamo.c, e.g.
*/
DECLARE_CXTSWPROT_VAR(mutex_t reset_pending_lock, INIT_LOCK_FREE(reset_pending_lock));
DECLARE_FREQPROT_VAR(uint reset_pending, 0);
* protected by allunits_lock since only read during unit creation.
*/
enum {
CACHE_BB = 0,
CACHE_TRACE,
CACHE_NUM_TYPES,
};
DECLARE_FREQPROT_VAR(uint reset_at_nth_unit[CACHE_NUM_TYPES],
{
0,
});
DECLARE_FREQPROT_VAR(uint reset_every_nth_unit[CACHE_NUM_TYPES],
{
0,
});
#define STATS_FCACHE_ADD(cache, stat, val) \
DOSTATS({ \
if (cache->is_shared) { \
if (cache->is_trace) \
STATS_ADD(fcache_shared_trace_##stat, val); \
else \
STATS_ADD(fcache_shared_bb_##stat, val); \
} else if (cache->is_trace) \
STATS_ADD(fcache_trace_##stat, val); \
else \
STATS_ADD(fcache_bb_##stat, val); \
})
#define STATS_FCACHE_SUB(cache, stat, val) \
STATS_FCACHE_ADD(cache, stat, -(stats_int_t)(val))
#define STATS_FCACHE_MAX(cache, stat1, stat2) \
DOSTATS({ \
if (cache->is_shared) { \
if (cache->is_trace) \
STATS_MAX(fcache_shared_trace_##stat1, fcache_shared_trace_##stat2); \
else \
STATS_MAX(fcache_shared_bb_##stat1, fcache_shared_bb_##stat2); \
} else if (cache->is_trace) \
STATS_MAX(fcache_trace_##stat1, fcache_trace_##stat2); \
else \
STATS_MAX(fcache_bb_##stat1, fcache_bb_##stat2); \
})
#ifdef DEBUG
# ifdef INTERNAL
static void
verify_fifo(dcontext_t *dcontext, fcache_t *cache);
static void
print_fifo(dcontext_t *dcontext, fcache_t *cache);
# endif
static void
fcache_cache_stats(dcontext_t *dcontext, fcache_t *cache);
#endif
static fcache_t *
fcache_cache_init(dcontext_t *dcontext, uint flags, bool initial_unit);
static void
fcache_cache_free(dcontext_t *dcontext, fcache_t *cache, bool free_units);
static void
add_to_free_list(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
fragment_t *f, cache_pc start_pc, uint size);
static void
fcache_free_unit(dcontext_t *dcontext, fcache_unit_t *unit, bool dealloc_or_reuse);
#define CHECK_PARAMS(who, name, ret) \
do { \
\
if (FCACHE_OPTION(cache_##who##_max) > 0 && \
FCACHE_OPTION(cache_##who##_max) < FCACHE_OPTION(cache_##who##_unit_max)) { \
FCACHE_OPTION(cache_##who##_unit_max) = FCACHE_OPTION(cache_##who##_max); \
FCACHE_OPTION(cache_##who##_unit_init) = FCACHE_OPTION(cache_##who##_max); \
FCACHE_OPTION(cache_##who##_unit_quadruple) = \
FCACHE_OPTION(cache_##who##_max); \
FCACHE_OPTION(cache_##who##_unit_upgrade) = \
FCACHE_OPTION(cache_##who##_max); \
} \
\
ret = check_param_bounds(&FCACHE_OPTION(cache_##who##_max), (uint)PAGE_SIZE, 0, \
name " cache max size") || \
ret; \
\
ret = check_param_bounds(&FCACHE_OPTION(cache_##who##_unit_max), \
FCACHE_OPTION(cache_##who##_unit_init), \
FCACHE_OPTION(cache_##who##_max), \
name " cache unit max size") || \
ret; \
ret = check_param_bounds(&FCACHE_OPTION(cache_##who##_unit_quadruple), \
FCACHE_OPTION(cache_##who##_unit_init), \
FCACHE_OPTION(cache_##who##_max), \
name " cache unit quadruple-to size") || \
ret; \
ret = check_param_bounds(&FCACHE_OPTION(cache_##who##_unit_upgrade), \
FCACHE_OPTION(cache_##who##_unit_init), \
FCACHE_OPTION(cache_##who##_max), \
name " cache unit free upgrade size") || \
ret; \
ret = \
check_param_bounds( \
&FCACHE_OPTION(cache_##who##_unit_init),
resizing fcache units */ \
IF_X64_ELSE(FCACHE_OPTION(cache_##who##_unit_max), (uint)PAGE_SIZE), \
FCACHE_OPTION(cache_##who##_unit_max), name " cache unit init size") || \
ret; \
\
\
} while (0);
#define CHECK_WSET_PARAM(param, ret) \
do { \
if (dynamo_options.cache_##param##_regen < 0) { \
USAGE_ERROR("-cache_" #param "_regen must be >= 0, is %d, setting to 0", \
dynamo_options.cache_##param##_regen); \
dynamo_options.cache_##param##_regen = 0; \
ret = true; \
} \
if (dynamo_options.cache_##param##_replace < 0) { \
USAGE_ERROR("-cache_" #param "_replace must be >= 0, id %d, setting to 0", \
dynamo_options.cache_##param##_replace); \
dynamo_options.cache_##param##_replace = 0; \
ret = true; \
} \
if (dynamo_options.cache_##param##_replace != 0 && \
dynamo_options.cache_##param##_regen > \
dynamo_options.cache_##param##_replace) { \
USAGE_ERROR("-cache_" #param "_regen (currently %d) must be <= " \
"-cache_" #param "_replace (currently %d) (if -cache_" #param \
"_replace > 0), setting regen to equal replace", \
dynamo_options.cache_##param##_regen, \
dynamo_options.cache_##param##_replace); \
dynamo_options.cache_##param##_regen = \
dynamo_options.cache_##param##_replace; \
ret = true; \
} \
} while (0);
* options, returns true if modified the value of any options to make them
* compatible. This is called while the options are writable. */
bool
fcache_check_option_compatibility()
{
bool ret = false;
uint i;
CHECK_PARAMS(bb, "Basic block", ret);
CHECK_PARAMS(trace, "Trace", ret);
CHECK_WSET_PARAM(bb, ret);
CHECK_WSET_PARAM(trace, ret);
if (DYNAMO_OPTION(shared_bbs)) {
if (DYNAMO_OPTION(cache_shared_bb_max) > 0) {
USAGE_ERROR("-cache_shared_bb_max != 0 not supported");
dynamo_options.cache_shared_bb_max = 0;
ret = true;
}
CHECK_PARAMS(shared_bb, "Shared bb", ret);
CHECK_WSET_PARAM(shared_bb, ret);
ret = check_param_bounds(&FCACHE_OPTION(cache_shared_bb_unit_init),
FCACHE_OPTION(cache_shared_bb_unit_max),
FCACHE_OPTION(cache_shared_bb_unit_max),
"cache_shared_bb_unit_init should equal "
"cache_shared_bb_unit_max") ||
ret;
}
if (DYNAMO_OPTION(shared_traces)) {
if (DYNAMO_OPTION(cache_shared_trace_max) > 0) {
USAGE_ERROR("-cache_shared_trace_max != 0 not supported");
dynamo_options.cache_shared_trace_max = 0;
ret = true;
}
CHECK_PARAMS(shared_trace, "Shared trace", ret);
CHECK_WSET_PARAM(shared_trace, ret);
ret = check_param_bounds(&FCACHE_OPTION(cache_shared_trace_unit_init),
FCACHE_OPTION(cache_shared_trace_unit_max),
FCACHE_OPTION(cache_shared_trace_unit_max),
"cache_shared_trace_unit_init should equal "
"cache_shared_trace_unit_max") ||
ret;
}
if (INTERNAL_OPTION(pad_jmps_shift_bb) &&
DYNAMO_OPTION(cache_bb_align) < START_PC_ALIGNMENT) {
USAGE_ERROR("if -pad_jmps_shift_bb, -cache_bb_align must be >= %d",
START_PC_ALIGNMENT);
dynamo_options.cache_bb_align = START_PC_ALIGNMENT;
ret = true;
}
if (INTERNAL_OPTION(pad_jmps_shift_trace) &&
DYNAMO_OPTION(cache_trace_align) < START_PC_ALIGNMENT) {
USAGE_ERROR("if -pad_jmps_shift_trace, -cache_trace_align must be >= %d",
START_PC_ALIGNMENT);
dynamo_options.cache_trace_align = START_PC_ALIGNMENT;
ret = true;
}
reset_at_nth_unit[CACHE_BB] = DYNAMO_OPTION(reset_at_nth_bb_unit);
reset_every_nth_unit[CACHE_BB] = DYNAMO_OPTION(reset_every_nth_bb_unit);
reset_at_nth_unit[CACHE_TRACE] = DYNAMO_OPTION(reset_at_nth_trace_unit);
reset_every_nth_unit[CACHE_TRACE] = DYNAMO_OPTION(reset_every_nth_trace_unit);
* after first at
*/
for (i = 0; i < CACHE_NUM_TYPES; i++) {
if (reset_every_nth_unit[i] > 0 && reset_at_nth_unit[i] == 0)
reset_at_nth_unit[i] = reset_every_nth_unit[i];
}
return ret;
}
static void
fcache_reset_init(void)
{
* FIXME: could set initial sizes to 0 for all configurations, instead
*/
if (RUNNING_WITHOUT_CODE_CACHE())
return;
if (DYNAMO_OPTION(shared_bbs)) {
shared_cache_bb = fcache_cache_init(GLOBAL_DCONTEXT, FRAG_SHARED, true);
ASSERT(shared_cache_bb != NULL);
LOG(GLOBAL, LOG_CACHE, 1, "Initial shared bb cache is %d KB\n",
shared_cache_bb->init_unit_size / 1024);
}
if (DYNAMO_OPTION(shared_traces)) {
shared_cache_trace =
fcache_cache_init(GLOBAL_DCONTEXT, FRAG_SHARED | FRAG_IS_TRACE, true);
ASSERT(shared_cache_trace != NULL);
LOG(GLOBAL, LOG_CACHE, 1, "Initial shared trace cache is %d KB\n",
shared_cache_trace->init_unit_size / 1024);
}
}
void
fcache_init()
{
ASSERT(offsetof(fragment_t, flags) == offsetof(empty_slot_t, flags));
DOCHECK(1, {
static free_list_header_t free;
free.flags = FRAG_FAKE | FRAG_FCACHE_FREE_LIST;
ASSERT(TEST(FRAG_FCACHE_FREE_LIST, ((fragment_t *)(&free))->flags));
ASSERT(offsetof(free_list_header_t, next) == offsetof(live_header_t, f));
});
ASSERT(FREE_LIST_SIZES[0] == 0);
VMVECTOR_ALLOC_VECTOR(fcache_unit_areas, GLOBAL_DCONTEXT,
VECTOR_SHARED | VECTOR_NEVER_MERGE, fcache_unit_areas);
allunits = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, fcache_list_t, ACCT_OTHER, PROTECTED);
allunits->units = NULL;
allunits->dead = NULL;
allunits->num_dead = 0;
allunits->units_to_flush = NULL;
allunits->units_to_free = NULL;
allunits->units_to_free_tail = NULL;
fcache_reset_init();
}
#ifdef WINDOWS_PC_SAMPLE
static void
fcache_unit_profile_stop(fcache_unit_t *u)
{
int sum;
stop_profile(u->profile);
sum = sum_profile(u->profile);
if (sum > 0) {
bool shared, trace;
if (u->cache == NULL) {
shared = u->was_shared;
trace = u->was_trace;
} else {
shared = u->cache->is_shared;
trace = u->cache->is_trace;
}
d_r_mutex_lock(&profile_dump_lock);
if (shared) {
print_file(profile_file,
"\nDumping fcache %s unit profile (Shared)\n%d hits\n",
trace ? "trace" : "bb", sum);
} else {
print_file(profile_file,
"\nDumping fcache %s unit profile (Thread %d)\n%d hits\n",
trace ? "trace" : "bb", u->dcontext->owning_thread, sum);
}
dump_profile(profile_file, u->profile);
d_r_mutex_unlock(&profile_dump_lock);
}
}
#endif
static inline void
remove_unit_from_cache(fcache_unit_t *u)
{
ASSERT(u->cache != NULL);
u->cache->size -= u->size;
RSTATS_DEC(fcache_num_live);
STATS_FCACHE_SUB(u->cache, capacity, u->size);
#ifdef WINDOWS_PC_SAMPLE
* until they are really freed, so we must cache their type here.
* We do need to clear their cache field to support private or other
* deletable flushable unit types (though w/ default ops today no flushable
* unit will have its cache deleted).
*/
u->was_trace = u->cache->is_trace;
u->was_shared = u->cache->is_shared;
#endif
u->cache = NULL;
}
static void
fcache_really_free_unit(fcache_unit_t *u, bool on_dead_list, bool dealloc_unit)
{
if (TEST(SELFPROT_CACHE, dynamo_options.protect_mask) && !u->writable) {
change_protection((void *)u->start_pc, u->size, WRITABLE);
}
#ifdef WINDOWS_PC_SAMPLE
if (u->profile != NULL) {
if (!on_dead_list)
fcache_unit_profile_stop(u);
free_profile(u->profile);
u->profile = NULL;
}
#endif
if (u->cache != NULL)
remove_unit_from_cache(u);
if (on_dead_list) {
ASSERT(u->cache == NULL);
allunits->num_dead--;
RSTATS_DEC(fcache_num_free);
STATS_SUB(fcache_free_capacity, u->size);
}
RSTATS_SUB(fcache_combined_capacity, u->size);
* being re-used and not showing up in in_fcache
*/
vmvector_remove(fcache_unit_areas, u->start_pc, u->reserved_end_pc);
if (dealloc_unit) {
heap_munmap((void *)u->start_pc, UNIT_RESERVED_SIZE(u),
VMM_CACHE | VMM_REACHABLE | (u->per_thread ? VMM_PER_THREAD : 0));
}
nonpersistent_heap_free(GLOBAL_DCONTEXT, u,
sizeof(fcache_unit_t) HEAPACCT(ACCT_MEM_MGT));
}
#ifdef DEBUG
void
fcache_stats_exit()
{
if (DYNAMO_OPTION(shared_bbs)) {
fcache_t *cache = shared_cache_bb;
if (cache != NULL) {
* dump_global_stats() which currently regularly calls
* this, so any ASSERTs on this path will deadlock
* (workaround is to be vigilant and use msgbox_mask).
*/
ASSERT_DO_NOT_OWN_MUTEX(cache->is_shared, &cache->lock);
PROTECT_CACHE(cache, lock);
fcache_cache_stats(GLOBAL_DCONTEXT, cache);
PROTECT_CACHE(cache, unlock);
}
}
if (DYNAMO_OPTION(shared_traces)) {
fcache_t *cache = shared_cache_trace;
if (cache != NULL) {
ASSERT_DO_NOT_OWN_MUTEX(cache->is_shared, &cache->lock);
PROTECT_CACHE(cache, lock);
fcache_cache_stats(GLOBAL_DCONTEXT, cache);
PROTECT_CACHE(cache, unlock);
}
}
}
#endif
* fcache_reset_init() will be called before continuing.
*/
static void
fcache_reset_free(void)
{
fcache_unit_t *u, *next_u;
* FIXME: could set initial sizes to 0 for all configurations, instead
*/
if (RUNNING_WITHOUT_CODE_CACHE())
return;
* fcache_really_free_unit() to move units onto dead list only to delete
* here: should directly delete, but maintain fcache stats.
*/
* synch issues (plus the lock will be deleted)
*/
if (DYNAMO_OPTION(shared_bbs)) {
fcache_cache_free(GLOBAL_DCONTEXT, shared_cache_bb, true);
shared_cache_bb = NULL;
}
if (DYNAMO_OPTION(shared_traces)) {
fcache_cache_free(GLOBAL_DCONTEXT, shared_cache_trace, true);
shared_cache_trace = NULL;
}
* we must free the units here as they are unreachable elsewhere.
* their fragments will be freed by the fragment htable walk.
*/
d_r_mutex_lock(&unit_flush_lock);
u = allunits->units_to_flush;
while (u != NULL) {
next_u = u->next_local;
LOG(GLOBAL, LOG_CACHE, 2,
"@ reset-free freeing to-be-flushed unit " PFX "-" PFX "\n", u->start_pc,
u->end_pc);
fcache_free_unit(GLOBAL_DCONTEXT, u, true);
u = next_u;
}
allunits->units_to_flush = NULL;
d_r_mutex_unlock(&unit_flush_lock);
ASSERT(allunits->units_to_free == NULL);
d_r_mutex_lock(&allunits_lock);
u = allunits->dead;
while (u != NULL) {
next_u = u->next_global;
fcache_really_free_unit(u, true , true );
u = next_u;
}
allunits->dead = NULL;
allunits->num_dead = 0;
d_r_mutex_unlock(&allunits_lock);
}
void
fcache_exit()
{
fcache_unit_t *u, *next_u;
DOSTATS({
LOG(GLOBAL, LOG_TOP | LOG_THREADS, 1, "fcache_exit: before fcache cleanup\n");
DOLOG(1, LOG_CACHE, fcache_stats_exit(););
});
fcache_reset_free();
d_r_mutex_lock(&allunits_lock);
u = allunits->units;
while (u != NULL) {
next_u = u->next_global;
fcache_really_free_unit(u, false , true );
u = next_u;
}
d_r_mutex_unlock(&allunits_lock);
ASSERT(vmvector_empty(fcache_unit_areas));
vmvector_delete_vector(GLOBAL_DCONTEXT, fcache_unit_areas);
HEAP_TYPE_FREE(GLOBAL_DCONTEXT, allunits, fcache_list_t, ACCT_OTHER, PROTECTED);
reset_pending = 0;
DELETE_LOCK(allunits_lock);
DELETE_LOCK(reset_pending_lock);
DELETE_LOCK(unit_flush_lock);
}
#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
void
fcache_profile_exit()
{
fcache_unit_t *u;
d_r_mutex_lock(&allunits_lock);
for (u = allunits->units; u != NULL; u = u->next_global) {
if (u->profile) {
fcache_unit_profile_stop(u);
free_profile(u->profile);
u->profile = NULL;
}
}
d_r_mutex_unlock(&allunits_lock);
}
#endif
static fcache_unit_t *
fcache_lookup_unit(cache_pc pc)
{
STATS_INC(fcache_unit_lookups);
return (fcache_unit_t *)vmvector_lookup(fcache_unit_areas, pc);
}
fragment_t *
fcache_fragment_pclookup(dcontext_t *dcontext, cache_pc lookup_pc, fragment_t *wrapper)
{
fragment_t *f, *found = NULL;
cache_pc pc;
fcache_unit_t *unit = fcache_lookup_unit(lookup_pc);
if (unit == NULL)
return NULL;
LOG(THREAD, LOG_CACHE, 5, "fcache_fragment_pclookup " PFX " -> " PFX "-" PFX "\n",
lookup_pc, unit->start_pc, unit->end_pc);
if (unit->cache->is_coarse) {
* to lock the cache itself.
*/
coarse_info_t *info = unit->cache->coarse_info;
cache_pc body;
app_pc tag;
ASSERT(info != NULL);
tag = fragment_coarse_pclookup(dcontext, info, lookup_pc, &body);
ASSERT(wrapper != NULL);
fragment_coarse_wrapper(wrapper, tag, body);
return wrapper;
}
PROTECT_CACHE(unit->cache, lock);
DODEBUG({
if (!unit->cache->consistent) {
* physically walk the cache. ASSUMPTION: this only happens for
* debug builds when we pclookup on disassembly.
*/
PROTECT_CACHE(unit->cache, unlock);
return fragment_pclookup_by_htable(dcontext, lookup_pc, wrapper);
}
});
pc = unit->start_pc;
while (pc < unit->cur_pc && pc < lookup_pc) {
f = *((fragment_t **)pc);
LOG(THREAD, LOG_CACHE, 6, "\treading " PFX " -> " PFX "\n", pc, f);
if (!USE_FIFO_FOR_CACHE(unit->cache)) {
if (FRAG_IS_FREE_LIST(f)) {
pc += ((free_list_header_t *)pc)->size;
continue;
}
}
ASSERT(f != NULL);
ASSERT(FIFO_UNIT(f) == unit);
ASSERT(FRAG_HDR_START(f) == pc);
if (!FRAG_EMPTY(f) && lookup_pc < (f->start_pc + f->size) &&
lookup_pc >= f->start_pc) {
found = f;
LOG(THREAD, LOG_CACHE, 5, "\tfound F%d (" PFX ")." PFX "\n", f->id, f->tag,
f->start_pc);
break;
}
pc += FRAG_SIZE(f);
}
PROTECT_CACHE(unit->cache, unlock);
return found;
}
dr_where_am_i_t
fcache_refine_whereami(dcontext_t *dcontext, dr_where_am_i_t whereami, app_pc pc,
OUT fragment_t **containing_fragment)
{
if (whereami != DR_WHERE_FCACHE) {
if (containing_fragment != NULL)
*containing_fragment = NULL;
return whereami;
}
fragment_t wrapper;
fragment_t *fragment = fragment_pclookup(dcontext, pc, &wrapper);
if (fragment == NULL) {
* XXX: we could double-check fcache_unit_areas.lock before
* calling (case 1317) and assert on it.
*/
if (in_fcache(pc)) {
whereami = DR_WHERE_UNKNOWN;
} else {
* It's all generated and post-process can't identify.
* Assume code order is as follows:
*/
if (in_indirect_branch_lookup_code(dcontext, (cache_pc)pc)) {
whereami = DR_WHERE_IBL;
} else if (in_generated_routine(dcontext, (cache_pc)pc)) {
* not just private or shared fcache_{enter,return} but also
* do_syscall and other common transition code.
*/
whereami = DR_WHERE_CONTEXT_SWITCH;
} else {
whereami = DR_WHERE_UNKNOWN;
}
}
}
if (containing_fragment != NULL)
*containing_fragment = fragment;
return whereami;
}
#ifdef DEBUG
static bool
fcache_pc_in_live_unit(fcache_t *cache, cache_pc pc)
{
fcache_unit_t *unit;
for (unit = cache->units; unit != NULL; unit = unit->next_local) {
if (pc >= unit->start_pc && pc < unit->end_pc)
return true;
}
unit = fcache_lookup_unit(pc);
* just prior to flush synch
*/
if (unit != NULL && unit->pending_flush)
return true;
return false;
}
#endif
bool
fcache_is_writable(fragment_t *f)
{
fcache_unit_t *unit = fcache_lookup_unit(f->start_pc);
ASSERT(unit != NULL);
return unit->writable;
}
* Else, does the unit f is part of
*/
void
fcache_change_fragment_protection(dcontext_t *dcontext, fragment_t *f, bool writable)
{
fcache_unit_t *u;
ASSERT(TEST(SELFPROT_CACHE, dynamo_options.protect_mask));
if (f != NULL) {
u = fcache_lookup_unit(f->start_pc);
ASSERT(u != NULL);
if (u->writable == writable)
return;
change_protection((void *)u->start_pc, u->size, writable);
u->writable = writable;
} else {
* win32 does not allow single protection change call on units that
* were allocated with separate calls so we don't try to combine
* adjacent units here
*/
* and the first segfaults
*/
d_r_mutex_lock(&allunits_lock);
u = allunits->units;
while (u != NULL) {
if (u->writable != writable) {
change_protection((void *)u->start_pc, u->size, writable);
u->writable = writable;
}
u = u->next_global;
}
d_r_mutex_unlock(&allunits_lock);
}
}
* Currently only the fcache_unit_areas write lock is checked, so if
* this routine is changed
* to grab any other locks, or call a routine that does, then the
* at_safe_spot() routine in os.c must be updated */
bool
in_fcache(void *pc)
{
return fcache_lookup_unit((cache_pc)pc) != NULL;
}
* If pc is non-NULL, this routine assumes that size is fully
* committed and initializes accordingly.
*/
static fcache_unit_t *
fcache_create_unit(dcontext_t *dcontext, fcache_t *cache, cache_pc pc, size_t size)
{
fcache_unit_t *u = NULL;
uint cache_type;
ASSERT(CACHE_PROTECTED(cache));
* contiguous empty slot that overflows a uint.
* minor: all our stats only take ints, as well
*/
ASSERT(CHECK_TRUNCATE_TYPE_uint(size));
ASSERT(ALIGNED(size, PAGE_SIZE));
if (pc == NULL) {
d_r_mutex_lock(&allunits_lock);
if (allunits->dead != NULL) {
fcache_unit_t *prev_u = NULL;
u = allunits->dead;
while (u != NULL) {
if (u->size >= size &&
(cache->max_size == 0 || cache->size + u->size <= cache->max_size)) {
if (prev_u == NULL)
allunits->dead = u->next_global;
else
prev_u->next_global = u->next_global;
LOG(THREAD, LOG_CACHE, 1,
"\tFound unit " PFX " of size %d (need %d) on dead list\n",
u->start_pc, u->size / 1024, size / 1024);
allunits->num_dead--;
RSTATS_DEC(fcache_num_free);
STATS_SUB(fcache_free_capacity, u->size);
#ifdef WINDOWS_PC_SAMPLE
if (u->profile) {
reset_profile(u->profile);
start_profile(u->profile);
}
#endif
break;
}
prev_u = u;
u = u->next_global;
}
}
d_r_mutex_unlock(&allunits_lock);
}
if (u == NULL) {
size_t commit_size;
u = (fcache_unit_t *)nonpersistent_heap_alloc(
GLOBAL_DCONTEXT, sizeof(fcache_unit_t) HEAPACCT(ACCT_MEM_MGT));
u->per_thread = false;
if (pc != NULL) {
u->start_pc = pc;
commit_size = size;
STATS_FCACHE_ADD(cache, claimed, size);
STATS_ADD(fcache_combined_claimed, size);
} else {
commit_size = DYNAMO_OPTION(cache_commit_increment);
* have a single param for the commit size, this
* makes it much easier to set it without having to
* set a dozen unrelated unit sizes too.
*/
if (commit_size > size)
commit_size = size;
which_vmm_t which = VMM_CACHE | VMM_REACHABLE;
if (!cache->is_shared && cache->units == NULL) {
* save space for many-threaded apps. These units are rarely used.
* We do not bother to mark subsequent units this way: the goal is to
* reduce up-front per-thread costs in common usage, while additional
* units indicate -thread_private or other settings.
*/
which |= VMM_PER_THREAD;
u->per_thread = true;
}
u->start_pc = (cache_pc)heap_mmap_reserve(
size, commit_size, MEMPROT_EXEC | MEMPROT_READ | MEMPROT_WRITE, which);
}
ASSERT(u->start_pc != NULL);
ASSERT(proc_is_cache_aligned((void *)u->start_pc));
LOG(THREAD, LOG_HEAP, 3, "fcache_create_unit -> " PFX "\n", u->start_pc);
u->size = commit_size;
u->end_pc = u->start_pc + commit_size;
u->reserved_end_pc = u->start_pc + size;
vmvector_add(fcache_unit_areas, u->start_pc, u->reserved_end_pc, (void *)u);
RSTATS_ADD_PEAK(fcache_combined_capacity, u->size);
#ifdef WINDOWS_PC_SAMPLE
if (dynamo_options.profile_pcs && dynamo_options.prof_pcs_fcache >= 2 &&
dynamo_options.prof_pcs_fcache <= 32) {
u->profile = create_profile(u->start_pc, u->reserved_end_pc,
dynamo_options.prof_pcs_fcache, NULL);
start_profile(u->profile);
} else
u->profile = NULL;
#endif
}
cache->size += u->size;
u->cur_pc = u->start_pc;
u->full = false;
u->cache = cache;
#if defined(SIDELINE) || defined(WINDOWS_PC_SAMPLE)
u->dcontext = dcontext;
#endif
u->writable = true;
u->pending_free = false;
DODEBUG({ u->pending_flush = false; });
u->flushtime = 0;
RSTATS_ADD_PEAK(fcache_num_live, 1);
STATS_FCACHE_ADD(u->cache, capacity, u->size);
STATS_FCACHE_MAX(u->cache, capacity_peak, capacity);
u->next_local = NULL;
d_r_mutex_lock(&allunits_lock);
if (allunits->units != NULL)
allunits->units->prev_global = u;
u->next_global = allunits->units;
u->prev_global = NULL;
allunits->units = u;
cache_type = cache->is_trace ? CACHE_TRACE : CACHE_BB;
if (reset_at_nth_unit[cache_type] > 0) {
* some were flushed, etc.)
*/
reset_at_nth_unit[cache_type]--;
if (reset_at_nth_unit[cache_type] == 0) {
schedule_reset(RESET_ALL);
if (reset_every_nth_unit[cache_type] > 0)
reset_at_nth_unit[cache_type] = reset_every_nth_unit[cache_type];
}
}
d_r_mutex_unlock(&allunits_lock);
return u;
}
static void
fcache_free_unit(dcontext_t *dcontext, fcache_unit_t *unit, bool dealloc_or_reuse)
{
DODEBUG({
if (unit->flushtime > 0) {
ASSERT_OWN_MUTEX(true, &unit_flush_lock);
* when freeing dead units -- not needed for release build
*/
unit->flushtime = 0;
} else {
ASSERT(dynamo_exited || dynamo_resetting || CACHE_PROTECTED(unit->cache));
}
});
d_r_mutex_lock(&allunits_lock);
if (unit->prev_global != NULL)
unit->prev_global->next_global = unit->next_global;
else
allunits->units = unit->next_global;
if (unit->next_global != NULL)
unit->next_global->prev_global = unit->prev_global;
STATS_FCACHE_SUB(unit->cache, claimed, (unit->cur_pc - unit->start_pc));
STATS_FCACHE_SUB(unit->cache, empty, (unit->cur_pc - unit->start_pc));
STATS_SUB(fcache_combined_claimed, (unit->cur_pc - unit->start_pc));
if (!dealloc_or_reuse) {
d_r_mutex_unlock(&allunits_lock);
fcache_really_free_unit(unit, false , false );
}
else if (allunits->num_dead < 5 ||
allunits->num_dead * 4U <= (uint)d_r_get_num_threads()) {
* when can take small and then needing to allocate when only
* have small left. Helps out with lots of small threads.
*/
fcache_unit_t *u, *prev_u;
for (u = allunits->dead, prev_u = NULL; u != NULL && u->size < unit->size;
prev_u = u, u = u->next_global)
;
unit->prev_global = NULL;
unit->next_local = NULL;
if (prev_u == NULL) {
unit->next_global = allunits->dead;
allunits->dead = unit;
} else {
unit->next_global = u;
prev_u->next_global = unit;
}
allunits->num_dead++;
RSTATS_ADD_PEAK(fcache_num_free, 1);
STATS_ADD(fcache_free_capacity, unit->size);
#ifdef WINDOWS_PC_SAMPLE
if (unit->profile)
fcache_unit_profile_stop(unit);
#endif
remove_unit_from_cache(unit);
d_r_mutex_unlock(&allunits_lock);
} else {
d_r_mutex_unlock(&allunits_lock);
fcache_really_free_unit(unit, false , true );
}
}
* larger-than-page block sizing (i#2607, i#4424). Guards will be added on top.
*/
#define SET_CACHE_PARAMS(cache, which) \
do { \
cache->max_size = FCACHE_OPTION(cache_##which##_max); \
cache->max_unit_size = FCACHE_OPTION(cache_##which##_unit_max); \
cache->max_quadrupled_unit_size = FCACHE_OPTION(cache_##which##_unit_quadruple); \
cache->free_upgrade_size = FCACHE_OPTION(cache_##which##_unit_upgrade); \
cache->init_unit_size = FCACHE_OPTION(cache_##which##_unit_init); \
cache->finite_cache = dynamo_options.finite_##which##_cache; \
cache->regen_param = dynamo_options.cache_##which##_regen; \
cache->replace_param = dynamo_options.cache_##which##_replace; \
} while (0);
static fcache_t *
fcache_cache_init(dcontext_t *dcontext, uint flags, bool initial_unit)
{
fcache_t *cache = (fcache_t *)nonpersistent_heap_alloc(
dcontext, sizeof(fcache_t) HEAPACCT(ACCT_MEM_MGT));
cache->fifo = NULL;
cache->size = 0;
cache->is_trace = TEST(FRAG_IS_TRACE, flags);
cache->is_shared = TEST(FRAG_SHARED, flags);
cache->is_coarse = TEST(FRAG_COARSE_GRAIN, flags);
DODEBUG({ cache->is_local = false; });
cache->coarse_info = NULL;
DODEBUG({ cache->consistent = true; });
if (cache->is_shared) {
ASSERT(dcontext == GLOBAL_DCONTEXT);
if (TEST(FRAG_IS_TRACE, flags)) {
DODEBUG({ cache->name = "Trace (shared)"; });
SET_CACHE_PARAMS(cache, shared_trace);
} else if (cache->is_coarse) {
DODEBUG({ cache->name = "Coarse basic block (shared)"; });
SET_CACHE_PARAMS(cache, coarse_bb);
} else {
DODEBUG({ cache->name = "Basic block (shared)"; });
SET_CACHE_PARAMS(cache, shared_bb);
}
} else {
ASSERT(dcontext != GLOBAL_DCONTEXT);
if (TEST(FRAG_IS_TRACE, flags)) {
DODEBUG({ cache->name = "Trace (private)"; });
SET_CACHE_PARAMS(cache, trace);
} else {
DODEBUG({ cache->name = "Basic block (private)"; });
SET_CACHE_PARAMS(cache, bb);
}
}
#ifdef DISALLOW_CACHE_RESIZING
cache->init_unit_size = cache->max_unit_size;
#endif
if (cache->is_shared)
ASSIGN_INIT_LOCK_FREE(cache->lock, shared_cache_lock);
if (initial_unit) {
PROTECT_CACHE(cache, lock);
cache->units = fcache_create_unit(dcontext, cache, NULL, cache->init_unit_size);
PROTECT_CACHE(cache, unlock);
} else
cache->units = NULL;
cache->num_regenerated = 0;
cache->num_replaced = 0;
cache->wset_check = 0;
cache->record_wset = false;
if (cache->is_shared) {
memset(cache->free_list, 0, sizeof(cache->free_list));
DODEBUG({
memset(cache->free_stats_freed, 0, sizeof(cache->free_stats_freed));
memset(cache->free_stats_reused, 0, sizeof(cache->free_stats_reused));
memset(cache->free_stats_coalesced, 0, sizeof(cache->free_stats_coalesced));
memset(cache->free_stats_charge, 0, sizeof(cache->free_stats_charge));
memset(cache->free_stats_split, 0, sizeof(cache->free_stats_split));
memset(cache->request_size_histogram, 0,
sizeof(cache->request_size_histogram));
memset(cache->free_size_histogram, 0, sizeof(cache->free_size_histogram));
});
}
return cache;
}
* If !free_units, we do not de-allocate or move the units to the dead list,
* but we still remove from the live list.
*/
static void
fcache_cache_free(dcontext_t *dcontext, fcache_t *cache, bool free_units)
{
fcache_unit_t *u, *next_u;
dcontext_t *alloc_dc = ALLOC_DC(dcontext, cache);
DEBUG_DECLARE(size_t cache_size = cache->size;)
DEBUG_DECLARE(size_t size_check = 0;)
if (USE_FIFO_FOR_CACHE(cache)) {
fragment_t *f, *nextf;
ASSERT(cache->consistent);
for (f = cache->fifo; f != NULL; f = nextf) {
* freed memory here since fragments will have been removed from FIFO
*/
nextf = FIFO_NEXT(f);
if (FRAG_EMPTY(f)) {
nonpersistent_heap_free(alloc_dc, f,
sizeof(empty_slot_t) HEAPACCT(ACCT_FCACHE_EMPTY));
}
}
cache->fifo = NULL;
}
ASSERT(cache->fifo == NULL);
u = cache->units;
while (u != NULL) {
DODEBUG({ size_check += u->size; });
next_u = u->next_local;
fcache_free_unit(dcontext, u, free_units);
u = next_u;
}
ASSERT(size_check == cache_size);
ASSERT(cache->size == 0);
if (cache->is_shared)
DELETE_LOCK(cache->lock);
nonpersistent_heap_free(alloc_dc, cache, sizeof(fcache_t) HEAPACCT(ACCT_MEM_MGT));
}
#ifdef DEBUG
void
fcache_free_list_consistency(dcontext_t *dcontext, fcache_t *cache, int bucket)
{
uint live = 0;
uint charge = 0;
uint waste = 0;
free_list_header_t *header = cache->free_list[bucket];
uint prev_size = 0;
fcache_unit_t *unit;
LOG(GLOBAL, LOG_CACHE, 3, "fcache_free_list_consistency %s bucket[%2d], %3d bytes\n",
cache->name, bucket, FREE_LIST_SIZES[bucket]);
while (header != NULL) {
cache_pc start_pc = (cache_pc)header;
uint size = header->size;
ASSERT(size >= FREE_LIST_SIZES[bucket] && size <= MAX_FREE_ENTRY_SIZE &&
(bucket == FREE_LIST_SIZES_NUM - 1 || size < FREE_LIST_SIZES[bucket + 1]));
* properly, when we start keeping them in order */
ASSERT(prev_size < size || true );
prev_size = size;
ASSERT(header->next == NULL || header->next->prev == header);
ASSERT(header->prev == NULL || header->prev->next == header);
ASSERT(FRAG_IS_FREE_LIST((fragment_t *)header));
unit = fcache_lookup_unit(start_pc);
if (unit->cur_pc > start_pc + header->size) {
fragment_t *subseq = FRAG_NEXT_SLOT(start_pc, header->size);
* followed by a marked fragment_t.
*/
ASSERT((!FRAG_IS_FREE_LIST(subseq) &&
TEST(FRAG_FOLLOWS_FREE_ENTRY, subseq->flags)) ||
* to ushort size limits */
size + FRAG_NEXT_FREE(start_pc, header->size)->size >
MAX_FREE_ENTRY_SIZE);
}
ASSERT(unit->full || unit->cur_pc != start_pc + header->size);
header = header->next;
* case 7318 was really wasting memory, for comparison here */
LOG(GLOBAL, LOG_CACHE, 4, "\t @" PFX ": %3d bytes, %3d max waste\n", start_pc,
size, size - FREE_LIST_SIZES[bucket]);
live++;
charge += size;
waste += size - FREE_LIST_SIZES[bucket];
}
ASSERT(live ==
cache->free_stats_freed[bucket] -
(cache->free_stats_reused[bucket] + cache->free_stats_coalesced[bucket]));
ASSERT(charge == cache->free_stats_charge[bucket]);
ASSERT(waste >= (
charged bytes - live entries * bucket _minimal_ size */
cache->free_stats_charge[bucket] -
(cache->free_stats_freed[bucket] -
(cache->free_stats_reused[bucket] +
cache->free_stats_coalesced[bucket])) *
FREE_LIST_SIZES[bucket]));
LOG(GLOBAL, LOG_CACHE, 2, "\t#%2d %3d bytes == %d live, %8d charge, %8d waste\n",
bucket, FREE_LIST_SIZES[bucket], live, charge, waste);
}
static void
fcache_cache_stats(dcontext_t *dcontext, fcache_t *cache)
{
fcache_unit_t *u;
int i = 0;
size_t capacity = 0;
size_t used = 0;
bool full = true;
ASSERT_OWN_MUTEX(!dynamo_exited && cache->is_shared, &cache->lock);
for (u = cache->units; u != NULL; u = u->next_local, i++) {
capacity += u->size;
used += u->cur_pc - u->start_pc;
full &= u->full;
LOG(THREAD, LOG_CACHE, 1,
"\t%s unit %d @" PFX ": capacity %d KB, used %d KB, %s\n", cache->name, i,
u->start_pc, u->size / 1024, (u->cur_pc - u->start_pc) / 1024,
u->full ? "full" : "not full");
}
LOG(THREAD, LOG_CACHE, 1, "%s cache: capacity %d KB, used %d KB, %s\n", cache->name,
capacity / 1024, used / 1024, full ? "full" : "not full");
if (DYNAMO_OPTION(cache_shared_free_list) && cache->is_shared) {
int bucket;
LOG(GLOBAL, LOG_CACHE, 1, "fcache %s free list stats:\n", cache->name);
for (bucket = 0; bucket < FREE_LIST_SIZES_NUM; bucket++) {
LOG(GLOBAL, LOG_ALL, 1,
"\t#%2d %3d bytes : %7d free, %7d reuse, %5d coalesce, %5d split\n"
"\t %3d bytes : %5d live, %8d charge, %8d waste\n",
bucket, FREE_LIST_SIZES[bucket], cache->free_stats_freed[bucket],
cache->free_stats_reused[bucket], cache->free_stats_coalesced[bucket],
cache->free_stats_split[bucket], FREE_LIST_SIZES[bucket],
cache->free_stats_freed[bucket] -
(cache->free_stats_reused[bucket] +
cache->free_stats_coalesced[bucket]),
cache->free_stats_charge[bucket],
cache->free_stats_charge[bucket] -
(cache->free_stats_freed[bucket] -
(cache->free_stats_reused[bucket] +
cache->free_stats_coalesced[bucket])) *
FREE_LIST_SIZES[bucket]);
}
DOLOG(1, LOG_CACHE, {
for (bucket = 0; bucket < FREE_LIST_SIZES_NUM; bucket++) {
fcache_free_list_consistency(dcontext, cache, bucket);
}
});
LOG(GLOBAL, LOG_ALL, 1, "fcache %s requests and frees histogram:\n", cache->name);
for (bucket = 0; bucket < sizeof(cache->request_size_histogram) /
sizeof(cache->request_size_histogram[0]);
bucket++) {
if (cache->request_size_histogram[bucket] != 0 ||
cache->free_size_histogram[bucket] != 0) {
LOG(GLOBAL, LOG_ALL, 1, "\t# %3d bytes == %8d requests %8d freed\n",
bucket * HISTOGRAM_GRANULARITY, cache->request_size_histogram[bucket],
cache->free_size_histogram[bucket]);
}
}
}
}
static inline int
get_histogram_bucket(int size)
{
uint histogram_bucket = size / HISTOGRAM_GRANULARITY;
if (histogram_bucket >= HISTOGRAM_MAX_SIZE / HISTOGRAM_GRANULARITY)
histogram_bucket = (HISTOGRAM_MAX_SIZE / HISTOGRAM_GRANULARITY) - 1;
return histogram_bucket;
}
#endif
static void
fcache_shift_fragments(dcontext_t *dcontext, fcache_unit_t *unit, ssize_t shift,
cache_pc start, cache_pc end, size_t old_size)
{
fragment_t *f;
cache_pc pc, new_pc;
ASSERT_OWN_MUTEX(unit->cache->is_shared, &unit->cache->lock);
ASSERT(!DYNAMO_OPTION(cache_shared_free_list) || !unit->cache->is_shared);
ASSERT(!unit->cache->is_coarse);
DODEBUG({ unit->cache->consistent = false; });
LOG(THREAD, LOG_CACHE, 2, "fcache_shift_fragments: first pass\n");
* fine to walk the old memory, we just need the fragment_t* pointers
*/
pc = unit->start_pc;
LOG(THREAD, LOG_CACHE, 2, " unit " PFX "-" PFX " [-" PFX "]\n", pc, unit->end_pc,
unit->reserved_end_pc);
while (pc < unit->cur_pc) {
f = *((fragment_t **)pc);
ASSERT(f != NULL);
ASSERT(FIFO_UNIT(f) == unit);
if (FRAG_EMPTY(f))
FRAG_START_ASSIGN(f, FRAG_START(f) + shift);
else {
LOG(THREAD, LOG_CACHE, 5, "\treading " PFX " -> " PFX " = F%d\n", pc, f,
f->id);
fragment_shift_fcache_pointers(dcontext, f, shift, start, end, old_size);
}
new_pc = FRAG_HDR_START(f);
LOG(THREAD, LOG_CACHE, 4, "resize: writing " PFX " to " PFX "\n", f, new_pc);
*((fragment_t **)vmcode_get_writable_addr(new_pc)) = f;
pc += FRAG_SIZE(f);
}
DOLOG(2, LOG_FRAGMENT, {
study_all_hashtables(dcontext);
});
LOG(THREAD, LOG_CACHE, 2, "fcache_shift_fragments: second pass\n");
pc = unit->start_pc;
LOG(THREAD, LOG_CACHE, 2, " unit " PFX "-" PFX " [-" PFX "]\n", pc, unit->end_pc,
unit->reserved_end_pc);
while (pc < unit->cur_pc) {
f = *((fragment_t **)pc);
ASSERT(f != NULL);
* f->start_pc, which is used to find the unit
*/
if (!FRAG_EMPTY(f)) {
LOG(THREAD, LOG_CACHE, 5, "\treading " PFX " -> " PFX " = F%d\n", pc, f,
f->id);
* we have links from bb cache to trace cache, and sometimes links
* the other direction, for example from a trace to a bb that
* cannot be a trace head (e.g., is marked CANNOT_BE_TRACE).
* simplest to re-link every fragment in the shifted cache.
* N.B.: we do NOT need to re-link the outgoing, since
* fragment_shift_fcache_pointers re-relativized all outgoing
* ctis by the shifted amount.
*/
if (TEST(FRAG_LINKED_INCOMING, f->flags)) {
unlink_fragment_incoming(dcontext, f);
link_fragment_incoming(dcontext, f, false );
}
}
pc += FRAG_SIZE(f);
}
DODEBUG({ unit->cache->consistent = true; });
}
static void
cache_extend_commitment(fcache_unit_t *unit, size_t commit_size)
{
ASSERT(unit != NULL);
ASSERT(ALIGNED(commit_size, DYNAMO_OPTION(cache_commit_increment)));
heap_mmap_extend_commitment(unit->end_pc, commit_size, VMM_CACHE | VMM_REACHABLE);
unit->end_pc += commit_size;
unit->size += commit_size;
unit->cache->size += commit_size;
unit->full = false;
STATS_FCACHE_ADD(unit->cache, capacity, commit_size);
STATS_FCACHE_MAX(unit->cache, capacity_peak, capacity);
RSTATS_ADD_PEAK(fcache_combined_capacity, commit_size);
ASSERT(unit->end_pc <= unit->reserved_end_pc);
ASSERT(unit->size <= UNIT_RESERVED_SIZE(unit));
}
* make the private-configuration caches larger. We could even get
* rid of the fcache shifting.
*/
* with labels-as-values since we can't patch those absolute addresses.
*/
static void
fcache_increase_size(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
uint slot_size)
{
bool reallocated = false;
cache_pc new_memory = NULL;
ssize_t shift;
size_t new_size = unit->size;
size_t commit_size;
ASSERT(!dr_bb_hook_exists() && !dr_trace_hook_exists());
ASSERT(unit->reserved_end_pc == unit->end_pc);
if (new_size * 4 <= cache->max_quadrupled_unit_size)
new_size *= 4;
else
new_size *= 2;
if (new_size < slot_size * MAX_SINGLE_MULTIPLE)
new_size = ALIGN_FORWARD(slot_size * MAX_SINGLE_MULTIPLE, PAGE_SIZE);
if (new_size > cache->max_unit_size)
new_size = cache->max_unit_size;
if (cache->max_size != 0 && cache->size - unit->size + new_size > cache->max_size) {
new_size = cache->max_size - cache->size + unit->size;
}
commit_size = new_size;
* allocated enough space for slot_size?
*/
ASSERT(unit->size + slot_size <= new_size);
LOG(THREAD, LOG_CACHE, 2, "Increasing %s unit size from %d KB to %d KB\n",
cache->name, unit->size / 1024, new_size / 1024);
#ifdef DISALLOW_CACHE_RESIZING
SYSLOG_INTERNAL_ERROR("This build cannot handle cache resizing");
ASSERT_NOT_REACHED();
#endif
ASSERT(CACHE_PROTECTED(cache));
if (allunits->dead != NULL) {
fcache_unit_t *u, *prev_u;
d_r_mutex_lock(&allunits_lock);
u = allunits->dead;
prev_u = NULL;
while (u != NULL) {
if (UNIT_RESERVED_SIZE(u) >= new_size) {
fcache_thread_units_t *tu =
(fcache_thread_units_t *)dcontext->fcache_field;
if (prev_u == NULL)
allunits->dead = u->next_global;
else
prev_u->next_global = u->next_global;
allunits->num_dead--;
RSTATS_DEC(fcache_num_free);
STATS_SUB(fcache_free_capacity, u->size);
LOG(THREAD, LOG_CACHE, 1, "\tFound unit of size %d on dead list\n",
u->size / 1024);
new_memory = u->start_pc;
ASSERT_TRUNCATE(new_size, uint, UNIT_RESERVED_SIZE(u));
new_size = UNIT_RESERVED_SIZE(u);
u->cache = cache;
* extension amount itself.
* Don't need to add to combined capacity: it includes free.
*/
STATS_FCACHE_ADD(cache, capacity, u->size);
STATS_FCACHE_MAX(cache, capacity_peak, capacity);
if (u->size < new_size) {
size_t new_commit = ALIGN_FORWARD(
new_size - u->size, DYNAMO_OPTION(cache_commit_increment));
ASSERT(u->size + new_commit <= UNIT_RESERVED_SIZE(u));
cache_extend_commitment(u, new_commit);
* cache_extend_commitment did */
u->cache->size -= new_commit;
ASSERT(u->size >= new_size);
}
commit_size = u->size;
ASSERT(tu->pending_unmap_pc == NULL);
tu->pending_unmap_pc = unit->start_pc;
tu->pending_unmap_size = UNIT_RESERVED_SIZE(unit);
STATS_FCACHE_SUB(cache, capacity, unit->size);
RSTATS_SUB(fcache_combined_capacity, unit->size);
#ifdef WINDOWS_PC_SAMPLE
if (u->profile != NULL) {
free_profile(u->profile);
u->profile = NULL;
}
#endif
* here and re-add down below
*/
vmvector_remove(fcache_unit_areas, u->start_pc, u->reserved_end_pc);
nonpersistent_heap_free(GLOBAL_DCONTEXT, u,
sizeof(fcache_unit_t) HEAPACCT(ACCT_MEM_MGT));
break;
}
prev_u = u;
u = u->next_global;
}
d_r_mutex_unlock(&allunits_lock);
}
if (new_memory == NULL) {
reallocated = true;
commit_size = 0;
while (commit_size < slot_size && unit->size + commit_size < new_size) {
commit_size += DYNAMO_OPTION(cache_commit_increment);
}
ASSERT(commit_size >= slot_size);
commit_size += unit->size;
ASSERT(commit_size <= new_size);
new_memory = (cache_pc)heap_mmap_reserve(
new_size, commit_size, MEMPROT_EXEC | MEMPROT_READ | MEMPROT_WRITE,
VMM_CACHE | VMM_REACHABLE);
STATS_FCACHE_SUB(cache, capacity, unit->size);
STATS_FCACHE_ADD(cache, capacity, commit_size);
STATS_FCACHE_MAX(cache, capacity_peak, capacity);
RSTATS_SUB(fcache_combined_capacity, unit->size);
RSTATS_ADD_PEAK(fcache_combined_capacity, commit_size);
LOG(THREAD, LOG_HEAP, 3, "fcache_increase_size -> " PFX "\n", new_memory);
ASSERT(new_memory != NULL);
ASSERT(proc_is_cache_aligned((void *)new_memory));
}
* the initial unit in a cache until it reaches the max unit size.
*/
ASSERT(unit == cache->units && unit->next_local == NULL);
memcpy(new_memory, unit->start_pc, unit->size);
* also update stored addresses like start pc
* assumption: all intra-cache links will still work!
* they're all relative, we copied entire cache!
*/
shift = (new_memory - unit->start_pc);
ASSERT(ALIGNED(shift, proc_get_cache_line_size()));
ASSERT(ALIGNED(shift, PAD_JMPS_ALIGNMENT));
fcache_shift_fragments(dcontext, unit, shift, new_memory, new_memory + new_size,
unit->size);
if (reallocated) {
* fcache_add_fragment, b/c the current ilist for being-added fragment
* may reference memory in old cache
*/
fcache_thread_units_t *tu = (fcache_thread_units_t *)dcontext->fcache_field;
ASSERT(tu->pending_unmap_pc == NULL);
tu->pending_unmap_pc = unit->start_pc;
tu->pending_unmap_size = UNIT_RESERVED_SIZE(unit);
}
* by the difference between new size and old size
*/
cache->size -= unit->size;
cache->size += commit_size;
unit->cur_pc += shift;
unit->start_pc = new_memory;
unit->size = commit_size;
unit->end_pc = unit->start_pc + commit_size;
unit->reserved_end_pc = unit->start_pc + new_size;
vmvector_add(fcache_unit_areas, unit->start_pc, unit->reserved_end_pc, (void *)unit);
unit->full = false;
#ifdef WINDOWS_PC_SAMPLE
{
* buffer to start of new buffer and maintain correspondence
*/
profile_t *old_prof = unit->profile;
if (old_prof != NULL) {
unit->profile = create_profile(unit->start_pc, unit->reserved_end_pc,
dynamo_options.prof_pcs_fcache, NULL);
stop_profile(old_prof);
ASSERT(unit->profile->buffer_size >= old_prof->buffer_size);
memcpy(unit->profile->buffer, old_prof->buffer, old_prof->buffer_size);
free_profile(old_prof);
start_profile(unit->profile);
}
}
#endif
DOLOG(2, LOG_CACHE, { verify_fifo(dcontext, cache); });
LOG(THREAD, LOG_CACHE, 1, "\tDone increasing unit size\n");
}
static void
fcache_thread_reset_init(dcontext_t *dcontext)
{
}
void
fcache_thread_init(dcontext_t *dcontext)
{
fcache_thread_units_t *tu = (fcache_thread_units_t *)heap_alloc(
dcontext, sizeof(fcache_thread_units_t) HEAPACCT(ACCT_OTHER));
dcontext->fcache_field = (void *)tu;
* this saves memory for both DYNAMO_OPTION(disable_traces) and for
* idle threads that never do much
*/
tu->trace = NULL;
* once we have that conditional for traces it's no extra cost for bbs
*/
tu->bb = NULL;
tu->pending_unmap_pc = NULL;
tu->pending_flush = false;
fcache_thread_reset_init(dcontext);
}
bool
fragment_lookup_deleted(dcontext_t *dcontext, app_pc tag)
{
future_fragment_t *fut;
if (SHARED_FRAGMENTS_ENABLED() && dcontext != GLOBAL_DCONTEXT) {
fut = fragment_lookup_private_future(dcontext, tag);
if (fut != NULL)
return TEST(FRAG_WAS_DELETED, fut->flags);
}
fut = fragment_lookup_future(dcontext, tag);
return (fut != NULL && TEST(FRAG_WAS_DELETED, fut->flags));
}
static future_fragment_t *
fragment_lookup_cache_deleted(dcontext_t *dcontext, fcache_t *cache, app_pc tag)
{
future_fragment_t *fut;
if (!cache->is_shared) {
* cache needs to be resized, and thus only if we kicked tag out of
* this cache, not whether we kicked it out of the shared cache
*/
fut = fragment_lookup_private_future(dcontext, tag);
} else
fut = fragment_lookup_future(dcontext, tag);
if (fut != NULL && TEST(FRAG_WAS_DELETED, fut->flags))
return fut;
else
return NULL;
}
#ifdef DEBUG
* be run before fragment_thread_exit, whereas the real fcache cleanup
* needs to be done after fragment's cleanup
*/
void
fcache_thread_exit_stats(dcontext_t *dcontext)
{
DOELOG(1, LOG_CACHE, {
fcache_thread_units_t *tu = (fcache_thread_units_t *)dcontext->fcache_field;
if (tu->bb != NULL)
fcache_cache_stats(dcontext, tu->bb);
if (tu->trace != NULL)
fcache_cache_stats(dcontext, tu->trace);
});
}
#endif
static void
fcache_thread_reset_free(dcontext_t *dcontext)
{
fcache_thread_units_t *tu = (fcache_thread_units_t *)dcontext->fcache_field;
if (tu->pending_unmap_pc != NULL) {
* being re-used and not showing up in in_fcache
*/
vmvector_remove(fcache_unit_areas, tu->pending_unmap_pc,
tu->pending_unmap_pc + tu->pending_unmap_size);
heap_munmap(tu->pending_unmap_pc, tu->pending_unmap_size,
VMM_CACHE | VMM_REACHABLE);
tu->pending_unmap_pc = NULL;
}
if (tu->bb != NULL) {
fcache_cache_free(dcontext, tu->bb, true);
tu->bb = NULL;
}
if (tu->trace != NULL) {
fcache_cache_free(dcontext, tu->trace, true);
tu->trace = NULL;
}
}
void
fcache_thread_exit(dcontext_t *dcontext)
{
DEBUG_DECLARE(fcache_thread_units_t *tu =
(fcache_thread_units_t *)dcontext->fcache_field;)
fcache_thread_reset_free(dcontext);
DODEBUG({
heap_free(dcontext, tu, sizeof(fcache_thread_units_t) HEAPACCT(ACCT_OTHER));
});
}
#if defined(DEBUG) && defined(INTERNAL)
static void
print_fifo(dcontext_t *dcontext, fcache_t *cache)
{
fragment_t *f = cache->fifo;
ASSERT(USE_FIFO_FOR_CACHE(cache));
ASSERT(CACHE_PROTECTED(cache));
while (f != NULL) {
LOG(THREAD, LOG_CACHE, 1, "\tF%d " PFX " = @" PFX " size %d\n", FRAG_ID(f),
FRAG_TAG(f), FRAG_HDR_START(f), FRAG_SIZE(f));
f = FIFO_NEXT(f);
}
}
static void
verify_fifo(dcontext_t *dcontext, fcache_t *cache)
{
fragment_t *f = cache->fifo;
fcache_unit_t *u;
cache_pc pc;
ASSERT(USE_FIFO_FOR_CACHE(cache));
ASSERT(CACHE_PROTECTED(cache));
while (f != NULL) {
LOG(THREAD, LOG_CACHE, 6, "\t*" PFX " F%d " PFX " = @" PFX " size %d\n", f,
FRAG_ID(f), FRAG_TAG(f), FRAG_HDR_START(f), FRAG_SIZE(f));
pc = FRAG_HDR_START(f);
ASSERT(*((fragment_t **)pc) == f);
pc += FRAG_SIZE(f);
u = FIFO_UNIT(f);
if ((!DYNAMO_OPTION(cache_shared_free_list) || !cache->is_shared)) {
if (pc < u->cur_pc)
ASSERT(*((fragment_t **)pc) != NULL);
}
f = FIFO_NEXT(f);
}
}
#endif
static void
fifo_append(fcache_t *cache, fragment_t *f)
{
ASSERT(USE_FIFO(f));
ASSERT(CACHE_PROTECTED(cache));
FIFO_NEXT_ASSIGN(f, NULL);
if (cache->fifo == NULL) {
cache->fifo = f;
FIFO_PREV_ASSIGN(f, f);
} else {
FIFO_PREV_ASSIGN(f, FIFO_PREV(cache->fifo));
FIFO_NEXT_ASSIGN(FIFO_PREV(cache->fifo), f);
FIFO_PREV_ASSIGN(cache->fifo, f);
}
FIFO_NEXT_ASSIGN(f, NULL);
LOG(THREAD_GET, LOG_CACHE, 5, "fifo_append F%d @" PFX "\n", FRAG_ID(f),
FRAG_HDR_START(f));
DOLOG(6, LOG_CACHE, { print_fifo(get_thread_private_dcontext(), cache); });
}
static void
fifo_remove(dcontext_t *dcontext, fcache_t *cache, fragment_t *f)
{
ASSERT(USE_FIFO(f));
ASSERT(CACHE_PROTECTED(cache));
ASSERT(cache->fifo != NULL);
if (f == cache->fifo) {
cache->fifo = FIFO_NEXT(f);
} else {
FIFO_NEXT_ASSIGN(FIFO_PREV(f), FIFO_NEXT(f));
}
if (FIFO_NEXT(f) == NULL) {
if (cache->fifo != NULL)
FIFO_PREV_ASSIGN(cache->fifo, FIFO_PREV(f));
} else {
FIFO_PREV_ASSIGN(FIFO_NEXT(f), FIFO_PREV(f));
}
LOG(THREAD, LOG_CACHE, 5, "fifo_remove F%d @" PFX "\n", FRAG_ID(f),
FRAG_HDR_START(f));
DOLOG(6, LOG_CACHE, { print_fifo(dcontext, cache); });
if (FRAG_EMPTY(f)) {
STATS_FCACHE_SUB(cache, empty, FRAG_SIZE(f));
STATS_FCACHE_ADD(cache, used, FRAG_SIZE(f));
nonpersistent_heap_free(ALLOC_DC(dcontext, cache), f,
sizeof(empty_slot_t) HEAPACCT(ACCT_FCACHE_EMPTY));
}
}
static void
fifo_prepend_empty(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
fragment_t *f, cache_pc start_pc, uint size)
{
empty_slot_t *slot;
ASSERT(CACHE_PROTECTED(cache));
STATS_FCACHE_ADD(cache, empty, size);
if (DYNAMO_OPTION(cache_shared_free_list) && cache->is_shared && !cache->is_coarse) {
ASSERT(USE_FREE_LIST_FOR_CACHE(cache));
add_to_free_list(dcontext, cache, unit, f, start_pc, size);
return;
}
* as there really is no alternative implemented -- we just waste the space.
* FIXME case 8714: anything we can do for coarse-grain?
*/
if (!USE_FIFO_FOR_CACHE(cache))
return;
* for efficiency only check front of FIFO -- most common case anyway
*/
if (cache->fifo != NULL && FRAG_EMPTY(cache->fifo)) {
if (FRAG_HDR_START(cache->fifo) == start_pc + size) {
LOG(THREAD, LOG_CACHE, 5, "prepend: just enlarging next empty\n");
FRAG_START_ASSIGN(cache->fifo, start_pc + HEADER_SIZE_FROM_CACHE(cache));
*((fragment_t **)vmcode_get_writable_addr(start_pc)) = cache->fifo;
FRAG_SIZE_ASSIGN(cache->fifo, FRAG_SIZE(cache->fifo) + size);
return;
} else if (FRAG_HDR_START(cache->fifo) + FRAG_SIZE(cache->fifo) == start_pc) {
LOG(THREAD, LOG_CACHE, 5, "prepend: just enlarging prev empty\n");
FRAG_SIZE_ASSIGN(cache->fifo, FRAG_SIZE(cache->fifo) + size);
return;
}
}
slot = (empty_slot_t *)nonpersistent_heap_alloc(
ALLOC_DC(dcontext, cache), sizeof(empty_slot_t) HEAPACCT(ACCT_FCACHE_EMPTY));
slot->flags = FRAG_FAKE | FRAG_IS_EMPTY_SLOT;
LOG(THREAD, LOG_CACHE, 5, "prepend: writing " PFX " to " PFX "\n", slot, start_pc);
*((empty_slot_t **)vmcode_get_writable_addr(start_pc)) = slot;
slot->start_pc = start_pc + HEADER_SIZE_FROM_CACHE(cache);
slot->fcache_size = size;
slot->next_fcache = cache->fifo;
if (cache->fifo == NULL)
slot->prev_fcache = (fragment_t *)slot;
else {
slot->prev_fcache = FIFO_PREV(cache->fifo);
FIFO_PREV_ASSIGN(cache->fifo, (fragment_t *)slot);
}
cache->fifo = (fragment_t *)slot;
LOG(THREAD, LOG_CACHE, 5, "fifo_prepend_empty F-1 @" PFX "\n", start_pc);
DOLOG(6, LOG_CACHE, { print_fifo(dcontext, cache); });
}
static bool
check_regen_replace_ratio(dcontext_t *dcontext, fcache_t *cache, uint add_size)
{
if (cache->max_size != 0 && cache->size + add_size > cache->max_size) {
LOG(THREAD, LOG_CACHE, 4, "at max size 0x%x\n", cache->max_size);
return false;
} else if (!cache->finite_cache || cache->replace_param == 0) {
LOG(THREAD, LOG_CACHE, 4, "upgrading since fcache_replace==0\n");
return true;
} else if (cache->regen_param == 0) {
LOG(THREAD, LOG_CACHE, 4, "will never upgrade since fcache_regen==0\n");
return false;
} else if (cache->wset_check > 0) {
ASSERT(USE_FIFO_FOR_CACHE(cache));
cache->wset_check--;
LOG(THREAD, LOG_CACHE, 4, "dec wset_check -> %d\n", cache->wset_check);
return false;
} else if (cache->size < cache->free_upgrade_size) {
if (USE_FIFO_FOR_CACHE(cache)) {
ASSERT(cache->wset_check == 0);
cache->wset_check = cache->replace_param;
} else {
ASSERT(!cache->is_coarse);
* start keeping track of regen stats
*/
if (cache->size + cache->max_unit_size >= cache->free_upgrade_size
* a larger unit than last new unit and drop back below threshold
*/
&& !cache->record_wset) {
cache->record_wset = true;
}
}
LOG(THREAD, LOG_CACHE, 3, "Free upgrade, no resize check\n");
return true;
} else {
if (USE_FIFO_FOR_CACHE(cache)) {
* to avoid too many checks when regen<<replace
*/
cache->wset_check = cache->replace_param;
} else {
ASSERT(!cache->is_coarse);
if (!cache->record_wset) {
* ideally we should hit this prior to the free upgrade point,
* as otherwise this is a 2nd free resize, but might not if
* create a new unit larger than max unit size.
*/
cache->record_wset = true;
return true;
}
}
if (cache->num_replaced >= cache->replace_param &&
cache->num_regenerated >= cache->regen_param) {
* and avoiding overflow (unless replace overflows before regen
* hits regen_param, which is very unlikely and who cares if it does)
*/
ASSERT(cache->replace_param > 0 && cache->regen_param > 0);
while (cache->num_replaced >= cache->replace_param &&
cache->num_regenerated >= cache->regen_param) {
cache->num_replaced -= cache->replace_param;
cache->num_regenerated -= cache->regen_param;
}
LOG(THREAD, LOG_CACHE, 3,
"Resize check: for %s unit: %d regenerated / %d replaced\n", cache->name,
cache->num_regenerated, cache->num_replaced);
if (cache->num_regenerated >= cache->regen_param) {
LOG(THREAD, LOG_CACHE, 1,
"%s unit reached ratio with %d regenerated / %d replaced\n",
cache->name, cache->num_regenerated, cache->num_replaced);
return true;
}
}
LOG(THREAD, LOG_CACHE, 4, "No resize allowed yet\n");
return false;
}
return true;
}
* If a small area is eaten and added to size, returns that amount
*/
static size_t
extend_unit_end(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit, size_t size,
bool rest_empty)
{
size_t left, extra = 0;
ASSERT(CACHE_PROTECTED(cache));
unit->cur_pc += size;
STATS_FCACHE_ADD(cache, claimed, size);
STATS_ADD(fcache_combined_claimed, size);
left = unit->end_pc - unit->cur_pc;
ASSERT(unit->end_pc >= unit->cur_pc);
if (left < MIN_UNIT_END_HOLE(cache)) {
LOG(THREAD_GET, LOG_CACHE, 3, "\tunit is now full\n");
unit->full = true;
if (left > 0) {
extra = left;
unit->cur_pc += extra;
STATS_FCACHE_ADD(cache, claimed, extra);
STATS_ADD(fcache_combined_claimed, extra);
}
} else if (rest_empty) {
ASSERT(CHECK_TRUNCATE_TYPE_uint(left));
fifo_prepend_empty(dcontext, cache, unit, NULL, unit->cur_pc, (uint)left);
unit->cur_pc += left;
STATS_FCACHE_ADD(cache, claimed, left);
STATS_ADD(fcache_combined_claimed, left);
unit->full = true;
}
LOG(THREAD, LOG_CACHE, 5, "\t\textend_unit_end: %d + %d / %d => cur_pc = " PFX "\n",
size, extra, left, unit->cur_pc);
* we cannot add it to stats b/c will never be removed!
*/
STATS_FCACHE_ADD(cache, used, (size + extra));
STATS_FCACHE_MAX(cache, peak, used);
return extra;
}
* For non-FIFO caches this routine cannot fail and must suspend the world
* and reset if necessary.
*/
static bool
try_for_more_space(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
uint slot_size)
{
size_t commit_size = DYNAMO_OPTION(cache_commit_increment);
ASSERT(CACHE_PROTECTED(cache));
if (unit->end_pc < unit->reserved_end_pc &&
!POINTER_OVERFLOW_ON_ADD(unit->cur_pc, slot_size) &&
!POINTER_OVERFLOW_ON_ADD(unit->end_pc, commit_size)) {
while (unit->cur_pc + slot_size > unit->end_pc + commit_size)
commit_size *= 2;
if (unit->end_pc + commit_size > unit->reserved_end_pc) {
commit_size = unit->reserved_end_pc - unit->end_pc;
}
cache_extend_commitment(unit, commit_size);
if (unit->cur_pc + slot_size > unit->end_pc) {
* still worth committing, we'll make an empty here
*/
extend_unit_end(dcontext, cache, unit, 0, true);
} else
return true;
}
if (cache->max_size == 0 || cache->size + slot_size <= cache->max_size) {
LOG(THREAD, LOG_CACHE, 1, "max size = %d, cur size = %d\n",
cache->max_size / 1024, cache->size / 1024);
* re-linking when resize.
* i#696: Don't try to resize fcache units when clients are present.
* They may use labels to insert absolute fragment PCs.
*/
if (unit->size >= cache->max_unit_size || dr_bb_hook_exists() ||
dr_trace_hook_exists()) {
fcache_unit_t *newunit;
size_t newsize;
ASSERT(!USE_FIFO_FOR_CACHE(cache) || cache->fifo != NULL ||
* fragment is >4KB! We'll have set wset_check though.
*/
cache->wset_check > 0);
extend_unit_end(dcontext, cache, unit, 0, true);
ASSERT(unit->full);
if (!USE_FIFO_FOR_CACHE(cache) && cache->finite_cache && !cache->is_coarse) {
* must grant the request, but if regen/replace ratio does not
* exceed target, must flush an old unit.
*/
if (!check_regen_replace_ratio(dcontext, cache,
0 )) {
fcache_thread_units_t *tu =
(fcache_thread_units_t *)dcontext->fcache_field;
fcache_unit_t *oldest = cache->units, *prev = NULL;
ASSERT(oldest != NULL);
for (; oldest->next_local != NULL;
prev = oldest, oldest = oldest->next_local)
;
* Flag will be cleared once really flushed in
* fcache_flush_pending_units()
*/
DODEBUG({ oldest->pending_flush = true; });
LOG(THREAD, LOG_CACHE, 2,
"marking unit " PFX "-" PFX " for flushing\n", oldest->start_pc,
oldest->end_pc);
if (prev == NULL)
cache->units = oldest->next_local;
else
prev->next_local = oldest->next_local;
if (unit == oldest)
unit = NULL;
d_r_mutex_lock(&unit_flush_lock);
oldest->next_local = allunits->units_to_flush;
allunits->units_to_flush = oldest;
STATS_ADD_PEAK(cache_units_toflush, 1);
* but we need the cache field for chain_fragments_for_flush() --
* so we assume for now that there are no deletable caches that
* don't use fifos yet are finite, and let
* append_units_to_free_list() remove from the cache later on.
* This does mean that cache->size is too big from now until
* then, so we don't really support hardcoded cache sizes.
*/
d_r_mutex_unlock(&unit_flush_lock);
tu->pending_flush = true;
STATS_INC(cache_units_wset_flushed);
} else
STATS_INC(cache_units_wset_allowed);
}
* if new frag is large, make unit large as well
*/
newsize = cache->max_unit_size;
if (newsize < slot_size * MAX_SINGLE_MULTIPLE)
newsize = ALIGN_FORWARD(slot_size * MAX_SINGLE_MULTIPLE, PAGE_SIZE);
if (cache->max_size > 0 && cache->size + newsize > cache->max_size)
newsize = cache->max_size - cache->size;
newunit = fcache_create_unit(dcontext, cache, NULL, newsize);
LOG(THREAD, LOG_CACHE, 1, "Creating a new %s unit of %d KB @" PFX "\n",
cache->name, newunit->size / 1024, newunit->start_pc);
newunit->next_local = cache->units;
cache->units = newunit;
} else {
ASSERT(!cache->is_coarse);
LOG(THREAD, LOG_CACHE, 1, "Increasing size of %s unit of %d KB @" PFX "\n",
cache->name, unit->size / 1024, unit->start_pc);
fcache_increase_size(dcontext, cache, unit, slot_size);
LOG(THREAD, LOG_CACHE, 1, "\tnow %d KB @" PFX "\n", unit->size / 1024,
unit->start_pc);
}
cache->num_replaced = 0;
cache->num_regenerated = 0;
DOLOG(2, LOG_CACHE, { fcache_cache_stats(dcontext, cache); });
return true;
} else {
if (!USE_FIFO_FOR_CACHE(cache)) {
ASSERT_NOT_REACHED();
* and aborts traces only of other threads (not this one, as this
* could be a trace we're emitting now). Then we could free all
* fragments in a unit here.
* In order to do the reset we'd first need to release cache->lock,
* if !cache->is_trace release the bb_building_lock, and enter_nolinking().
* Note that for bb cache we could instead do a full reset and then
* transfer_to_dispatch() but in debug builds we won't
* free locals in prior stack frames: the fragment_t, the
* instrlist_t, etc. For trace cache doing that would be a bigger
* step backward and take longer to get back here.
*/
}
* FIXME: but if trace cache has small max size, should just
* not build traces that big!
*/
if (cache->max_size > 0 && slot_size > (int)cache->max_size) {
#ifdef INTERNAL
const char *name = "";
DODEBUG(name = cache->name;);
#endif
USAGE_ERROR("single %s fragment (%d bytes) > max cache size (%d bytes)", name,
slot_size, cache->max_size);
}
return false;
}
}
static void
place_fragment(dcontext_t *dcontext, fragment_t *f, fcache_unit_t *unit,
cache_pc header_pc)
{
fcache_t *cache = unit->cache;
ASSERT_OWN_MUTEX(unit->cache->is_shared, &unit->cache->lock);
DOLOG(3, LOG_CACHE,
{
fragment_t wrapper;
ASSERT(fragment_pclookup_by_htable(dcontext, header_pc + HEADER_SIZE(f),
&wrapper) == NULL);
});
if (HEADER_SIZE(f) > 0) {
LOG(THREAD, LOG_CACHE, 5, "place: writing " PFX " to " PFX "\n", f, header_pc);
*((fragment_t **)vmcode_get_writable_addr(header_pc)) = f;
}
* guy needs no padding at start
*/
FRAG_START_ASSIGN(f, header_pc + HEADER_SIZE(f));
ASSERT(ALIGNED(FRAG_HDR_START(f), SLOT_ALIGNMENT(cache)));
STATS_FCACHE_ADD(cache, headers, HEADER_SIZE(f));
STATS_FCACHE_ADD(cache, align, f->fcache_extra - (stats_int_t)HEADER_SIZE(f));
if (cache->record_wset) {
* may have replaced the future? xref case 7151, though that
* should be a problem for private as well...
*/
future_fragment_t *fut = fragment_lookup_cache_deleted(dcontext, cache, f->tag);
ASSERT(!USE_FIFO_FOR_CACHE(cache));
ASSERT(!cache->is_coarse);
cache->num_replaced++;
if (fut != NULL) {
cache->num_regenerated++;
STATS_INC(num_fragments_regenerated);
SHARED_FLAGS_RECURSIVE_LOCK(fut->flags, acquire, change_linking_lock);
fut->flags &= ~FRAG_WAS_DELETED;
SHARED_FLAGS_RECURSIVE_LOCK(fut->flags, release, change_linking_lock);
}
LOG(THREAD, LOG_CACHE, 4, "For %s unit: %d regenerated / %d replaced\n",
cache->name, cache->num_regenerated, cache->num_replaced);
}
}
#ifdef DEBUG
static void
removed_fragment_stats(dcontext_t *dcontext, fcache_t *cache, fragment_t *f)
{
int prefixes = fragment_prefix_size(f->flags);
linkstub_t *l = FRAGMENT_EXIT_STUBS(f);
int stubs = 0;
uint selfmod = 0;
* option the fragment will be obliterated at this point and we will not
* be able to locate the stub pc for an indirect exit.
* Instead we simply calculate what the stub sizes should be.
*/
for (; l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
int sz;
if (!EXIT_HAS_LOCAL_STUB(l->flags, f->flags))
continue;
sz = linkstub_size(dcontext, f, l);
if (LINKSTUB_INDIRECT(l->flags))
STATS_FCACHE_ADD(cache, indirect_stubs, -sz);
else {
ASSERT(LINKSTUB_DIRECT(l->flags));
STATS_FCACHE_ADD(cache, direct_stubs, -sz);
}
stubs += sz;
}
if (TEST(FRAG_SELFMOD_SANDBOXED, f->flags)) {
* changed (selfmod doesn't make page RO!), so we use a stored size
* that's there just for stats
*/
selfmod = FRAGMENT_SELFMOD_COPY_SIZE(f);
STATS_FCACHE_SUB(cache, selfmod_copy, selfmod);
}
STATS_FCACHE_SUB(cache, bodies, (f->size - (prefixes + stubs + selfmod)));
STATS_FCACHE_SUB(cache, prefixes, prefixes);
STATS_FCACHE_SUB(cache, headers, HEADER_SIZE(f));
STATS_FCACHE_SUB(cache, align, (f->fcache_extra - (stats_int_t)HEADER_SIZE(f)));
}
#endif
static void
force_fragment_from_cache(dcontext_t *dcontext, fcache_t *cache, fragment_t *victim)
{
bool empty = FRAG_EMPTY(victim);
ASSERT(CACHE_PROTECTED(cache));
if (USE_FIFO(victim))
fifo_remove(dcontext, cache, victim);
if (!empty) {
* when it makes a future fragment it uses the FRAG_WAS_DELETED flag
*/
if (cache->finite_cache)
cache->num_replaced++;
DOSTATS({ removed_fragment_stats(dcontext, cache, victim); });
STATS_INC(num_fragments_replaced);
fragment_delete(dcontext, victim, FRAGDEL_NO_FCACHE);
}
}
static bool
replace_fragments(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
fragment_t *f, fragment_t *fifo, uint slot_size)
{
fragment_t *victim;
uint slot_so_far;
cache_pc pc, header_pc;
ASSERT(CACHE_PROTECTED(cache));
ASSERT(!DYNAMO_OPTION(cache_shared_free_list) || !cache->is_shared);
ASSERT(USE_FIFO_FOR_CACHE(cache));
DODEBUG({ cache->consistent = false; });
slot_so_far = 0;
pc = FRAG_HDR_START(fifo);
victim = fifo;
while (true) {
if (TEST(FRAG_CANNOT_DELETE, victim->flags)) {
DODEBUG({ cache->consistent = true; });
return false;
}
slot_so_far += FRAG_SIZE(victim);
if (slot_so_far >= slot_size)
break;
pc += FRAG_SIZE(victim);
if (pc == unit->cur_pc) {
break;
}
ASSERT(pc < unit->cur_pc);
victim = *((fragment_t **)pc);
LOG(THREAD, LOG_CACHE, 5, "\treading " PFX " -> " PFX "\n", pc, victim);
ASSERT(victim != NULL);
ASSERT(FIFO_UNIT(victim) == unit);
}
LOG(THREAD, LOG_CACHE, 4, "\treplacing fragment(s) in filled unit\n");
header_pc = FRAG_HDR_START(fifo);
slot_so_far = 0;
pc = header_pc;
victim = fifo;
while (true) {
slot_so_far += FRAG_SIZE(victim);
pc += FRAG_SIZE(victim);
LOG(THREAD, LOG_CACHE, 4, "\t\tdeleting F%d => %d bytes\n", FRAG_ID(victim),
slot_so_far);
force_fragment_from_cache(dcontext, cache, victim);
if (slot_so_far >= slot_size)
break;
* assumption: wouldn't be here if not enough victims below us, so
* don't need to check for end of cache
*/
if (pc == unit->cur_pc) {
size_t extra =
extend_unit_end(dcontext, cache, unit, slot_size - slot_so_far, false);
LOG(THREAD, LOG_CACHE, 4, "\t\textending unit by %d => %d bytes\n",
slot_size - slot_so_far, slot_size);
ASSERT(CHECK_TRUNCATE_TYPE_uint(extra));
FRAG_SIZE_ASSIGN(f, slot_size + (uint)extra);
slot_so_far = slot_size;
break;
}
ASSERT(pc < unit->cur_pc);
victim = *((fragment_t **)pc);
}
if (slot_so_far > slot_size) {
uint diff = slot_so_far - slot_size;
* free entry and if so coalescing any size space with it
*/
if (diff < MIN_EMPTY_HOLE(cache)) {
FRAG_SIZE_ASSIGN(f, slot_so_far);
LOG(THREAD, LOG_CACHE, 4, "\t\teating extra %d bytes\n", diff);
} else {
fifo_prepend_empty(dcontext, cache, unit, NULL, header_pc + FRAG_SIZE(f),
diff);
STATS_FCACHE_SUB(cache, used, diff);
}
}
place_fragment(dcontext, f, unit, header_pc);
fifo_append(cache, f);
if (cache->finite_cache && cache->num_replaced > 0) {
future_fragment_t *fut = fragment_lookup_cache_deleted(dcontext, cache, f->tag);
ASSERT(cache->finite_cache && cache->replace_param > 0);
if (fut != NULL) {
cache->num_regenerated++;
STATS_INC(num_fragments_regenerated);
SHARED_FLAGS_RECURSIVE_LOCK(fut->flags, acquire, change_linking_lock);
fut->flags &= ~FRAG_WAS_DELETED;
SHARED_FLAGS_RECURSIVE_LOCK(fut->flags, release, change_linking_lock);
}
LOG(THREAD, LOG_CACHE, 4, "For %s unit: %d regenerated / %d replaced\n",
cache->name, cache->num_regenerated, cache->num_replaced);
}
DODEBUG({ cache->consistent = true; });
return true;
}
static inline bool
replace_fifo(dcontext_t *dcontext, fcache_t *cache, fragment_t *f, uint slot_size,
fragment_t *fifo)
{
fcache_unit_t *unit;
ASSERT(USE_FIFO(f));
ASSERT(CACHE_PROTECTED(cache));
while (fifo != NULL) {
unit = FIFO_UNIT(fifo);
if ((ptr_uint_t)(unit->end_pc - FRAG_HDR_START(fifo)) >= slot_size) {
* could fail if un-deletable frags
*/
DOLOG(4, LOG_CACHE, { verify_fifo(dcontext, cache); });
if (replace_fragments(dcontext, cache, unit, f, fifo, slot_size))
return true;
}
fifo = FIFO_NEXT(fifo);
}
return false;
}
static inline int
find_free_list_bucket(uint size)
{
int bucket;
for (bucket = FREE_LIST_SIZES_NUM - 1; size < FREE_LIST_SIZES[bucket]; bucket--)
;
ASSERT(bucket >= 0);
return bucket;
}
static inline free_list_footer_t *
free_list_footer_from_header(free_list_header_t *h)
{
return (free_list_footer_t *)(((cache_pc)h) + h->size - sizeof(free_list_footer_t));
}
static inline free_list_header_t *
free_list_header_from_footer(free_list_footer_t *h)
{
return (free_list_header_t *)(((cache_pc)h) + sizeof(free_list_footer_t) - h->size);
}
static inline void
remove_from_free_list(fcache_t *cache, uint bucket,
free_list_header_t *header _IF_DEBUG(bool coalesce))
{
ASSERT(CACHE_PROTECTED(cache));
ASSERT(DYNAMO_OPTION(cache_shared_free_list) && cache->is_shared);
LOG(GLOBAL, LOG_CACHE, 4, "remove_from_free_list: %s bucket[%d] %d bytes @" PFX "\n",
cache->name, bucket, header->size, header);
if (header->prev != NULL) {
free_list_header_t *prev_writable =
(free_list_header_t *)vmcode_get_writable_addr((byte *)header->prev);
prev_writable->next = header->next;
} else
cache->free_list[bucket] = header->next;
if (header->next != NULL) {
free_list_header_t *next_writable =
(free_list_header_t *)vmcode_get_writable_addr((byte *)header->next);
next_writable->prev = header->prev;
}
* follows header.
* no reason to remove FRAG_FCACHE_FREE_LIST flag here.
*/
DOSTATS({
if (coalesce)
cache->free_stats_coalesced[bucket]++;
else
cache->free_stats_reused[bucket]++;
cache->free_stats_charge[bucket] -= header->size;
});
}
static void
add_to_free_list(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
fragment_t *f, cache_pc start_pc, uint size)
{
int bucket;
free_list_header_t *header = (free_list_header_t *)start_pc;
ASSERT(CACHE_PROTECTED(cache));
ASSERT(DYNAMO_OPTION(cache_shared_free_list) && cache->is_shared);
DODEBUG({
if (f != NULL)
cache->free_size_histogram[get_histogram_bucket(size)]++;
});
DOCHECK(CHKLVL_DEFAULT,
{
ASSERT(dynamo_resetting || fcache_pc_in_live_unit(cache, start_pc));
});
if (size > MAX_FREE_ENTRY_SIZE) {
* we can't: we would need to split into two empty slots.
* For now we bail and leak.
*/
ASSERT_NOT_REACHED();
return;
}
if (unit->cur_pc > start_pc + size && !POINTER_OVERFLOW_ON_ADD(start_pc, size)) {
fragment_t *subseq = FRAG_NEXT_SLOT(start_pc, size);
if (FRAG_IS_FREE_LIST(subseq)) {
free_list_header_t *next_header = FRAG_NEXT_FREE(start_pc, size);
if ((uint)next_header->size + size <= MAX_FREE_ENTRY_SIZE) {
uint next_bucket = find_free_list_bucket(next_header->size);
LOG(GLOBAL, LOG_CACHE, 4,
"add_to_free_list: coalesce w/ next %s bucket[%d] %d bytes @" PFX
"\n",
cache->name, next_bucket, next_header->size, next_header);
size += next_header->size;
remove_from_free_list(cache, next_bucket,
next_header _IF_DEBUG(true ));
* (potentially coalesce with prev as well)
*/
STATS_FCACHE_ADD(cache, free_coalesce_next, 1);
} else {
* free the whole unit.
*/
STATS_FCACHE_ADD(cache, free_coalesce_too_big, 1);
}
} else {
* This is the only place we need to mark, as we disallow free lists at
* the end of the current unit (so an appended fragment_t will never need
* this flag).
*/
LOG(GLOBAL, LOG_CACHE, 4,
"add_to_free_list: marking next F%d(" PFX ")." PFX " as after-free\n",
subseq->id, subseq->tag, subseq->start_pc);
ASSERT(FIFO_UNIT(subseq) == unit);
ASSERT(FRAG_HDR_START(subseq) == start_pc + size);
if (!TEST(FRAG_FOLLOWS_FREE_ENTRY, subseq->flags)) {
if (TEST(FRAG_SHARED, subseq->flags))
acquire_recursive_lock(&change_linking_lock);
subseq->flags |= FRAG_FOLLOWS_FREE_ENTRY;
if (TEST(FRAG_SHARED, subseq->flags))
release_recursive_lock(&change_linking_lock);
}
}
}
* Other reasons to come here should never have a free entry in the prev slot.
*/
if (f != NULL && TEST(FRAG_FOLLOWS_FREE_ENTRY, f->flags)) {
free_list_footer_t *prev_footer =
(free_list_footer_t *)(start_pc - sizeof(free_list_footer_t));
free_list_header_t *prev_header = free_list_header_from_footer(prev_footer);
uint prev_bucket = find_free_list_bucket(prev_footer->size);
uint new_size = (uint)prev_header->size + size;
if (new_size <= MAX_FREE_ENTRY_SIZE) {
LOG(GLOBAL, LOG_CACHE, 4,
"add_to_free_list: coalesce w/ prev %s bucket[%d] %d bytes @" PFX "\n",
cache->name, prev_bucket, prev_header->size, prev_header);
size += prev_header->size;
header = prev_header;
start_pc = (cache_pc)header;
remove_from_free_list(cache, prev_bucket,
prev_header _IF_DEBUG(true ));
STATS_FCACHE_ADD(cache, free_coalesce_prev, 1);
} else {
STATS_FCACHE_ADD(cache, free_coalesce_too_big, 1);
}
}
* It we want to relax this we must mark an appended fragment as
* FRAG_FOLLOWS_FREE_ENTRY. We do allow a free entry at the very end of the
* now-full cur unit. FIXME: this code is fragile wrt extend_unit_end's
* fifo_prepend_empty(), which wants a free list at the end of the unit, and
* only avoids disaster here by not incrementing cur_pc until afterward, so
* our condition here is not triggered. We could add another param to
* distinguish (f==NULL is not good enough as splits also call here).
*/
if (unit == cache->units && unit->cur_pc == start_pc + size) {
ASSERT((unit->full && unit->cur_pc == unit->end_pc) ||
(!unit->full && unit->cur_pc < unit->end_pc));
unit->cur_pc = start_pc;
unit->full = false;
STATS_FCACHE_ADD(cache, return_last, 1);
STATS_FCACHE_SUB(cache, claimed, size);
STATS_SUB(fcache_combined_claimed, size);
return;
}
* but not if it's to be its own entry
*/
ASSERT(size >= MIN_FCACHE_SLOT_SIZE(cache));
bucket = find_free_list_bucket(size);
free_list_header_t *header_writable =
(free_list_header_t *)vmcode_get_writable_addr((byte *)header);
header_writable->next = cache->free_list[bucket];
header_writable->prev = NULL;
header_writable->size = size;
header_writable->flags = FRAG_FAKE | FRAG_FCACHE_FREE_LIST;
free_list_footer_t *footer_writable = free_list_footer_from_header(header_writable);
footer_writable->size = size;
if (cache->free_list[bucket] != NULL) {
ASSERT(cache->free_list[bucket]->prev == NULL);
free_list_header_t *list_writable =
(free_list_header_t *)vmcode_get_writable_addr(
(byte *)cache->free_list[bucket]);
list_writable->prev = header;
}
cache->free_list[bucket] = header;
DOSTATS({
cache->free_stats_freed[bucket]++;
cache->free_stats_charge[bucket] += size;
LOG(GLOBAL, LOG_CACHE, 4, "add_to_free_list: %s bucket[%d] %d bytes @" PFX "\n",
cache->name, bucket, size, start_pc);
});
}
static bool
find_free_list_slot(dcontext_t *dcontext, fcache_t *cache, fragment_t *f, uint size)
{
int bucket;
cache_pc start_pc;
uint free_size;
free_list_header_t *header = NULL;
fcache_unit_t *unit = NULL;
DEBUG_DECLARE(bool split_empty = false;)
ASSERT(!USE_FIFO(f) && USE_FREE_LIST(f));
ASSERT(CACHE_PROTECTED(cache));
ASSERT(DYNAMO_OPTION(cache_shared_free_list) && cache->is_shared);
LOG(THREAD, LOG_CACHE, 4, "find_free_list_slot: %d bytes\n", size);
DODEBUG({ cache->request_size_histogram[get_histogram_bucket(size)]++; });
if (size > MAX_FREE_ENTRY_SIZE) {
return false;
}
* found, then upgrade. Coalescing and splitting allows a blind
* always-upgrade strategy.
*/
* FIXME: for any bucket if we are
* too close to the top we should look in the next bucket? (We
* used to have a scheme for FREE_LIST_BOTTOM_BUCKET_MARGIN)
*/
for (bucket = find_free_list_bucket(size); bucket < FREE_LIST_SIZES_NUM; bucket++) {
if (cache->free_list[bucket] == NULL)
continue;
* finish immediately */
header = cache->free_list[bucket];
while (header != NULL && header->size < size) {
* much space by picking the first large enough slot */
* fragment while walking list and make sure that it is
* coalesced */
header = header->next;
}
if (header != NULL)
break;
}
if (bucket >= FREE_LIST_SIZES_NUM)
return false;
DOSTATS({
if (bucket > find_free_list_bucket(size))
STATS_FCACHE_ADD(cache, free_use_larger, 1);
});
ASSERT(header != NULL);
remove_from_free_list(cache, bucket, header _IF_DEBUG(false ));
start_pc = (cache_pc)header;
free_size = header->size;
ASSERT(free_size >= size);
ASSERT(free_size <= MAX_FREE_ENTRY_SIZE);
* unit ptr/tag in the free header.*/
unit = fcache_lookup_unit(start_pc);
ASSERT(unit != NULL);
DOCHECK(CHKLVL_DEFAULT, {
ASSERT(fcache_pc_in_live_unit(cache, start_pc));
});
* MIN_EMPTY_HOLE() this will also kick in. Currently an
* issue only for traces and the bucket [112, 172).
*/
if (free_size - size > MIN_EMPTY_HOLE(cache)) {
DODEBUG({
cache->free_stats_split[bucket]++;
split_empty = true;
});
STATS_FCACHE_ADD(cache, free_split, 1);
add_to_free_list(dcontext, cache, unit, NULL, start_pc + size, free_size - size);
free_size = size;
} else {
if (unit->cur_pc > start_pc + free_size) {
fragment_t *subseq = FRAG_NEXT_SLOT(start_pc, free_size);
* if it exists
*/
if (!FRAG_IS_FREE_LIST(subseq)) {
LOG(GLOBAL, LOG_CACHE, 4,
"find_free_list_slot: un-marking next F%d(" PFX ")." PFX
" as after-free\n",
subseq->id, subseq->tag, subseq->start_pc);
ASSERT(FIFO_UNIT(subseq) == unit);
ASSERT(FRAG_HDR_START(subseq) == start_pc + free_size);
if (TEST(FRAG_SHARED, subseq->flags))
acquire_recursive_lock(&change_linking_lock);
ASSERT(TEST(FRAG_FOLLOWS_FREE_ENTRY, subseq->flags));
subseq->flags &= ~FRAG_FOLLOWS_FREE_ENTRY;
if (TEST(FRAG_SHARED, subseq->flags))
release_recursive_lock(&change_linking_lock);
} else {
* unable to coalesce due to ushort size limits
*/
ASSERT(free_size + FRAG_NEXT_FREE(start_pc, free_size)->size >
MAX_FREE_ENTRY_SIZE);
}
}
}
place_fragment(dcontext, f, unit, start_pc);
FRAG_SIZE_ASSIGN(f, free_size);
DOSTATS({
LOG(GLOBAL, LOG_CACHE, 4,
"find_free_list_slot: %s bucket[%d]%s"
" %d bytes @" PFX " requested %d, waste=%d\n",
cache->name, bucket, split_empty ? "split" : "", free_size, start_pc, size,
(free_size - size));
});
STATS_FCACHE_ADD(cache, align, free_size - size);
STATS_FCACHE_SUB(cache, empty, free_size);
return true;
}
static void
add_fragment_common(dcontext_t *dcontext, fcache_t *cache, fragment_t *f, uint slot_size)
{
fragment_t *fifo = NULL;
fcache_unit_t *unit;
ASSERT(CACHE_PROTECTED(cache));
* don't look for empty of appropriate size -- kick out the neighbors!
* we've found that works better, else empty list too long, and splitting
* it by size ruins the fifo
*/
if (USE_FREE_LIST_FOR_CACHE(cache)) {
if (DYNAMO_OPTION(cache_shared_free_list) &&
find_free_list_slot(dcontext, cache, f, slot_size))
return;
} else if (USE_FIFO_FOR_CACHE(cache)) {
fifo = cache->fifo;
while (fifo != NULL && FRAG_EMPTY(fifo)) {
unit = FIFO_UNIT(fifo);
if ((ptr_uint_t)(unit->end_pc - FRAG_HDR_START(fifo)) >= slot_size) {
* could fail if un-deletable frags
*/
LOG(THREAD, LOG_CACHE, 4, "\ttrying to fit in empty slot\n");
DOLOG(4, LOG_CACHE, { verify_fifo(dcontext, cache); });
if (replace_fragments(dcontext, cache, unit, f, fifo, slot_size))
return;
}
fifo = FIFO_NEXT(fifo);
}
}
unit = cache->units;
if (!unit->full && (ptr_uint_t)(unit->end_pc - unit->cur_pc) >= slot_size) {
size_t extra;
place_fragment(dcontext, f, unit, unit->cur_pc);
extra = extend_unit_end(dcontext, cache, unit, slot_size, false);
if (extra > 0) {
ASSERT(CHECK_TRUNCATE_TYPE_uint(extra));
FRAG_SIZE_ASSIGN(f, slot_size + (uint)extra);
STATS_FCACHE_ADD(cache, align, extra);
}
if (USE_FIFO_FOR_CACHE(cache))
fifo_append(cache, f);
LOG(THREAD, LOG_CACHE, 4,
"\tadded F%d to unfilled unit @" PFX " (%d [/%d] bytes left now)\n", f->id,
f->start_pc, unit->end_pc - unit->cur_pc, UNIT_RESERVED_SIZE(unit));
return;
}
* for fifo caches, don't resize unless regen/replace ratio warrants it.
*/
if (!USE_FIFO_FOR_CACHE(cache) || cache->is_coarse ||
check_regen_replace_ratio(dcontext, cache, slot_size)) {
LOG(THREAD, LOG_CACHE, 3, "\tcache is full, trying to acquire more space\n");
if (try_for_more_space(dcontext, cache, unit, slot_size)) {
add_fragment_common(dcontext, cache, f, slot_size);
return;
}
}
* been resized in step 3)
*/
ASSERT(USE_FIFO_FOR_CACHE(cache));
* not enough room from victim to end of cache.
* fifo should be pointing to first non-empty slot!
*/
if (replace_fifo(dcontext, cache, f, slot_size, fifo))
return;
LOG(THREAD, LOG_CACHE, 3,
"\ttried to avoid resizing, but no way around it for fragment size %d\n",
slot_size);
if (try_for_more_space(dcontext, cache, unit, slot_size)) {
add_fragment_common(dcontext, cache, f, slot_size);
return;
}
* plus some undeletable fragments right in the middle.
* abort the trace, hopefully making fragments re-deletable, and try again.
*/
trace_abort(dcontext);
if (replace_fifo(dcontext, cache, f, slot_size, cache->fifo))
return;
* current impl only makes trace-in-progress undeletable, right?
* so should never get here
*/
ASSERT_NOT_REACHED();
}
void
fcache_shift_start_pc(dcontext_t *dcontext, fragment_t *f, uint space)
{
fcache_unit_t *unit;
fcache_t *cache = NULL;
if (space == 0)
return;
* once that fragment is in a cache, as contig-cache-walkers need a consistent view!
*/
if (TEST(FRAG_SHARED, f->flags)) {
* this assumes that no cache lock is used for private fragments!
*/
unit = FIFO_UNIT(f);
ASSERT(unit != NULL);
cache = unit->cache;
PROTECT_CACHE(cache, lock);
}
ASSERT(ALIGNED(f->start_pc, START_PC_ALIGNMENT));
ASSERT(space <= START_PC_ALIGNMENT - 1);
ASSERT(PAD_JMPS_SHIFT_START(f->flags));
* if it's valid instructions */
SET_TO_DEBUG(vmcode_get_writable_addr(f->start_pc), space);
f->start_pc += space;
ASSERT_TRUNCATE(f->size, ushort, (f->size - space));
f->size = (ushort)(f->size - space);
DODEBUG({
if (space > 0) {
STATS_PAD_JMPS_ADD(f->flags, num_start_pc_shifted, 1);
STATS_PAD_JMPS_ADD(f->flags, start_pc_shifted_bytes, space);
}
});
if (TEST(FRAG_SHARED, f->flags))
PROTECT_CACHE(cache, unlock);
}
void
fcache_return_extra_space(dcontext_t *dcontext, fragment_t *f, size_t space_in)
{
fcache_unit_t *unit = FIFO_UNIT(f);
fcache_t *cache = unit->cache;
uint returnable_space, slot_padding, space;
cache_pc end_addr;
uint min_padding = 0;
byte *returnable_start;
bool released = false;
PROTECT_CACHE(cache, lock);
ASSERT_TRUNCATE(space, uint, space_in);
space = (uint)space_in;
DOSTATS({
if (ALIGN_FORWARD(f->size + HEADER_SIZE(f) - space, SLOT_ALIGNMENT(cache)) <
MIN_FCACHE_SLOT_SIZE(cache))
STATS_INC(num_final_fragment_too_small);
});
* any end padding (for slot alignment etc., stored in fcache->extra) */
returnable_space = space + f->fcache_extra - HEADER_SIZE(f);
if (FRAG_SIZE(f) - returnable_space < MIN_FCACHE_SLOT_SIZE(cache)) {
min_padding = returnable_space - (FRAG_SIZE(f) - MIN_FCACHE_SLOT_SIZE(cache));
returnable_space = FRAG_SIZE(f) - MIN_FCACHE_SLOT_SIZE(cache);
}
end_addr = FRAG_HDR_START(f) + FRAG_SIZE(f) - returnable_space;
ASSERT_TRUNCATE(slot_padding, uint,
ALIGN_FORWARD(end_addr, SLOT_ALIGNMENT(cache)) -
(ptr_uint_t)end_addr);
slot_padding =
(uint)(ALIGN_FORWARD(end_addr, SLOT_ALIGNMENT(cache)) - (ptr_uint_t)end_addr);
returnable_space -= slot_padding;
returnable_start = FRAG_HDR_START(f) + FRAG_SIZE(f) - returnable_space;
ASSERT(FRAG_SIZE(f) - returnable_space >= MIN_FCACHE_SLOT_SIZE(cache));
ASSERT(ALIGNED(returnable_space, SLOT_ALIGNMENT(cache)));
ASSERT(ALIGNED(returnable_start, SLOT_ALIGNMENT(cache)));
if (returnable_space > 0) {
STATS_INC(pad_jmps_fragments_overestimated);
if (FRAG_HDR_START(f) + FRAG_SIZE(f) == unit->end_pc) {
ASSERT(unit->full);
if (returnable_space >= MIN_UNIT_END_HOLE(cache)) {
if (unit == cache->units) {
ASSERT(unit->cur_pc == unit->end_pc);
unit->full = false;
unit->cur_pc -= returnable_space;
STATS_FCACHE_SUB(cache, claimed, returnable_space);
STATS_SUB(fcache_combined_claimed, returnable_space);
released = true;
} else {
fifo_prepend_empty(dcontext, cache, unit, NULL, returnable_start,
returnable_space);
released = true;
}
}
} else if (FRAG_HDR_START(f) + FRAG_SIZE(f) == unit->cur_pc) {
ASSERT(!unit->full);
unit->cur_pc -= returnable_space;
STATS_FCACHE_SUB(cache, claimed, returnable_space);
STATS_SUB(fcache_combined_claimed, returnable_space);
released = true;
} else {
if (!(DYNAMO_OPTION(cache_shared_free_list) && cache->is_shared)) {
fragment_t *next_f;
* be a fragment or empty slot after us */
next_f = FRAG_NEXT_SLOT(FRAG_HDR_START(f), FRAG_SIZE(f));
ASSERT(next_f != NULL);
if (FRAG_EMPTY(next_f)) {
STATS_FCACHE_ADD(cache, empty, returnable_space);
FRAG_START_ASSIGN(next_f, returnable_start + HEADER_SIZE(f));
*((fragment_t **)vmcode_get_writable_addr(returnable_start)) = next_f;
FRAG_SIZE_ASSIGN(next_f, FRAG_SIZE(next_f) + returnable_space);
released = true;
}
}
if (!released) {
* if excess is large enough by itself
*/
fragment_t *subseq = FRAG_NEXT_SLOT(FRAG_HDR_START(f), FRAG_SIZE(f));
ASSERT(FRAG_HDR_START(f) + FRAG_SIZE(f) < unit->cur_pc);
if ((FRAG_IS_FREE_LIST(subseq) &&
* FIXME: fragile if coalesce rules change -- perhaps have
* free list routine return failure?
*/
returnable_space +
FRAG_NEXT_FREE(FRAG_HDR_START(f), FRAG_SIZE(f))->size <=
MAX_FREE_ENTRY_SIZE) ||
returnable_space >= MIN_EMPTY_HOLE(cache)) {
fifo_prepend_empty(dcontext, cache, unit, NULL, returnable_start,
returnable_space);
released = true;
DOSTATS({
if (FRAG_IS_FREE_LIST(subseq))
STATS_INC(pad_jmps_excess_next_free);
});
}
}
}
}
* from f->size to f->fcache_extra
*/
ASSERT(f->fcache_extra + space ==
slot_padding + HEADER_SIZE(f) + returnable_space + min_padding);
ASSERT_TRUNCATE(f->size, ushort, (f->size - space));
f->size = (ushort)(f->size - space);
FRAG_SIZE_ASSIGN(f,
f->size + HEADER_SIZE(f) + FRAG_START_PADDING(f) + slot_padding +
(released ? 0 : returnable_space) + min_padding);
ASSERT(FRAG_SIZE(f) >= MIN_FCACHE_SLOT_SIZE(cache));
DOSTATS({
if (released)
STATS_FCACHE_SUB(cache, used, returnable_space);
else if (returnable_space > 0) {
STATS_INC(pad_jmps_excess_wasted);
STATS_ADD(pad_jmps_excess_wasted_bytes, returnable_space);
}
});
DOLOG(3, LOG_CACHE, {
if (USE_FIFO_FOR_CACHE(cache))
verify_fifo(dcontext, cache);
});
PROTECT_CACHE(cache, unlock);
STATS_PAD_JMPS_ADD(f->flags, extra_space_released, released ? returnable_space : 0);
}
static fcache_t *
get_cache_for_new_fragment(dcontext_t *dcontext, fragment_t *f)
{
fcache_thread_units_t *tu = (fcache_thread_units_t *)dcontext->fcache_field;
fcache_t *cache = NULL;
if (TEST(FRAG_SHARED, f->flags)) {
if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
coarse_info_t *info = get_executable_area_coarse_info(f->tag);
ASSERT(info != NULL);
if (info != NULL && info->frozen)
info = info->non_frozen;
ASSERT(info != NULL);
if (info->cache == NULL) {
* before acquiring the info->lock
*/
cache = fcache_cache_init(GLOBAL_DCONTEXT, f->flags, true);
d_r_mutex_lock(&info->lock);
if (info->cache == NULL) {
cache->coarse_info = info;
coarse_unit_init(info, cache);
ASSERT(cache == info->cache);
d_r_mutex_unlock(&info->lock);
} else {
ASSERT_CURIOSITY(false && "race in creating coarse cache");
d_r_mutex_unlock(&info->lock);
fcache_cache_free(GLOBAL_DCONTEXT, cache, true);
}
}
ASSERT(info->cache != NULL);
ASSERT(((fcache_t *)info->cache)->coarse_info == info);
return (fcache_t *)info->cache;
} else {
if (IN_TRACE_CACHE(f->flags))
return shared_cache_trace;
else
return shared_cache_bb;
}
} else {
if (IN_TRACE_CACHE(f->flags)) {
if (tu->trace == NULL) {
tu->trace = fcache_cache_init(dcontext, FRAG_IS_TRACE, true);
ASSERT(tu->trace != NULL);
LOG(THREAD, LOG_CACHE, 1, "Initial trace cache is %d KB\n",
tu->trace->init_unit_size / 1024);
}
return tu->trace;
} else {
if (tu->bb == NULL) {
tu->bb = fcache_cache_init(dcontext, 0 , true);
ASSERT(tu->bb != NULL);
LOG(THREAD, LOG_CACHE, 1, "Initial basic block cache is %d KB\n",
tu->bb->init_unit_size / 1024);
}
return tu->bb;
}
}
ASSERT_NOT_REACHED();
return NULL;
}
void
fcache_add_fragment(dcontext_t *dcontext, fragment_t *f)
{
fcache_thread_units_t *tu = (fcache_thread_units_t *)dcontext->fcache_field;
uint slot_size;
fcache_t *cache = get_cache_for_new_fragment(dcontext, f);
ASSERT(cache != NULL);
PROTECT_CACHE(cache, lock);
f->start_pc = (cache_pc)(ptr_uint_t)START_PC_ALIGNMENT;
if (tu->pending_unmap_pc != NULL) {
* being re-used and not showing up in in_fcache
*/
vmvector_remove(fcache_unit_areas, tu->pending_unmap_pc,
tu->pending_unmap_pc + tu->pending_unmap_size);
heap_munmap(tu->pending_unmap_pc, tu->pending_unmap_size,
VMM_CACHE | VMM_REACHABLE);
tu->pending_unmap_pc = NULL;
}
* be cache-line-aligned. we use a 4-byte header as a backpointer
* to the fragment_t.
*/
slot_size = f->size + HEADER_SIZE(f);
if (slot_size < MIN_FCACHE_SLOT_SIZE(cache)) {
STATS_INC(num_fragment_too_small);
slot_size = MIN_FCACHE_SLOT_SIZE(cache);
}
slot_size = (uint)ALIGN_FORWARD(slot_size, SLOT_ALIGNMENT(cache));
ASSERT(slot_size >= f->size + HEADER_SIZE(f));
ASSERT_TRUNCATE(f->fcache_extra, byte, (slot_size - f->size));
f->fcache_extra = (byte)(slot_size - f->size);
LOG(THREAD, LOG_CACHE, 4,
"fcache_add_fragment to %s cache (size %dKB): F%d w/ size %d (=> %d)\n",
cache->name, cache->units->size / 1024, f->id, f->size, slot_size);
add_fragment_common(dcontext, cache, f, slot_size);
ASSERT(!PAD_JMPS_SHIFT_START(f->flags) ||
ALIGNED(f->start_pc, START_PC_ALIGNMENT));
DOLOG(3, LOG_CACHE, {
if (USE_FIFO_FOR_CACHE(cache))
verify_fifo(dcontext, cache);
});
PROTECT_CACHE(cache, unlock);
}
void
fcache_remove_fragment(dcontext_t *dcontext, fragment_t *f)
{
fcache_unit_t *unit = FIFO_UNIT(f);
fcache_t *cache = unit->cache;
ASSERT(dcontext != GLOBAL_DCONTEXT || dynamo_exited || dynamo_resetting ||
TEST(FRAG_WAS_DELETED, f->flags) || is_self_allsynch_flushing());
PROTECT_CACHE(cache, lock);
LOG(THREAD, LOG_CACHE, 4, "fcache_remove_fragment: F%d from %s unit\n", f->id,
cache->name);
DOSTATS({ removed_fragment_stats(dcontext, cache, f); });
STATS_FCACHE_SUB(cache, used, FRAG_SIZE(f));
#ifdef DEBUG_MEMORY
* We do this before fifo_prepend_empty to avoid figuring whether
* to leave alone th1 st 4 bytes of fragment space or not (it's
* used to store the size for cache_shared_free_list).
* FIXME: put in the rest of the patterns and checks to make this
* parallel to heap DEBUG_MEMORY (==case 5657)
*/
memset(vmcode_get_writable_addr(f->start_pc), DEBUGGER_INTERRUPT_BYTE, f->size);
#endif
* the unit on free lists or the FIFO
*/
if (!unit->pending_free) {
fifo_prepend_empty(dcontext, cache, unit, f, FRAG_HDR_START(f), FRAG_SIZE(f));
if (USE_FIFO(f)) {
fifo_remove(dcontext, cache, f);
DOLOG(3, LOG_CACHE, { verify_fifo(dcontext, cache); });
}
}
PROTECT_CACHE(cache, unlock);
}
#ifdef SIDELINE
dcontext_t *
get_dcontext_for_fragment(fragment_t *f)
{
fcache_unit_t *unit = fcache_lookup_unit(f->start_pc);
ASSERT(unit != NULL);
return unit->dcontext;
}
#endif
* FLUSHING OF UNITS
*
* General strategy: first, put units to flush on the global units_to_flush
* list. At a nolinking point, use flush synch to get other threads out of
* the target units, and then walk each unit using contiguous header walk.
* Remove free list and lazy entries, and chain up all fragments to pass to
* flush unlink_shared(). Move units to a global units_to_free list, recording
* their flushtimes. We'll be notified when a pending delete entry is
* freed, and we can then check which units are safe to free.
*/
bool
fcache_is_flush_pending(dcontext_t *dcontext)
{
fcache_thread_units_t *tu = (fcache_thread_units_t *)dcontext->fcache_field;
return tu->pending_flush;
}
* caller must set pending_free and flushtime fields.
*/
static void
append_units_to_free_list(fcache_unit_t *u)
{
d_r_mutex_lock(&unit_flush_lock);
if (allunits->units_to_free_tail == NULL) {
ASSERT(allunits->units_to_free == NULL);
allunits->units_to_free = u;
} else {
ASSERT(allunits->units_to_free->flushtime <= u->flushtime);
ASSERT(allunits->units_to_free_tail->next_local == NULL);
ASSERT(allunits->units_to_free_tail != u);
allunits->units_to_free_tail->next_local = u;
}
STATS_ADD_PEAK(cache_units_tofree, 1);
ASSERT(u->flushtime > 0);
ASSERT(u->pending_free);
while (u->next_local != NULL) {
if (u->cache != NULL)
remove_unit_from_cache(u);
ASSERT(u->cache == NULL);
STATS_ADD_PEAK(cache_units_tofree, 1);
ASSERT(u->flushtime <= u->next_local->flushtime);
u = u->next_local;
ASSERT(u->flushtime > 0);
ASSERT(u->pending_free);
}
allunits->units_to_free_tail = u;
ASSERT(allunits->units_to_free_tail->next_local == NULL);
d_r_mutex_unlock(&unit_flush_lock);
}
* unit into a list, by only calling us between stage1 and stage2 of
* flushing (there's no other synch that is safe).
*/
static fragment_t *
chain_fragments_for_flush(dcontext_t *dcontext, fcache_unit_t *unit, fragment_t **tail)
{
fragment_t *list = NULL, *f = NULL, *prev_f = NULL;
cache_pc pc;
ASSERT(is_self_flushing());
LOG(THREAD, LOG_CACHE, 4, "\tchaining fragments in unit " PFX "-" PFX "\n",
unit->start_pc, unit->end_pc);
* vm_area_remove_fragment(), and do another complete walk in
* unlink_fragments_for_deletion() -- can we reduce to one walk?
*/
pc = unit->start_pc;
while (pc < unit->cur_pc) {
bool add_to_list = false;
f = *((fragment_t **)pc);
LOG(THREAD, LOG_CACHE, 5, "\treading " PFX " -> " PFX "\n", pc, f);
if (USE_FREE_LIST_FOR_CACHE(unit->cache)) {
if (FRAG_IS_FREE_LIST(f)) {
* remove this entry from the free list
*/
free_list_header_t *cur_free = ((free_list_header_t *)pc);
int bucket = find_free_list_bucket(cur_free->size);
LOG(THREAD, LOG_CACHE, 5,
"\tremoving free list entry " PFX " size %d bucket [%d]\n", cur_free,
cur_free->size, bucket);
* flush synch is enough. we can't hold cache lock for this
* entire routine b/c it has lower rank than the lazy_delete lock.
*/
PROTECT_CACHE(unit->cache, lock);
remove_from_free_list(unit->cache, bucket,
cur_free _IF_DEBUG(false ));
PROTECT_CACHE(unit->cache, unlock);
pc += cur_free->size;
continue;
}
}
ASSERT(f != NULL);
ASSERT(FIFO_UNIT(f) == unit);
ASSERT(FRAG_HDR_START(f) == pc);
ASSERT(!TEST(FRAG_CANNOT_DELETE, f->flags));
if (TEST(FRAG_WAS_DELETED, f->flags)) {
* while typically it will be deleted before the other fragments
* in this unit (since flushtime now is < ours, and lazy are deleted
* before pending-delete), there is a case where it will not be:
* if the lazy list hits its threshold and a new pending-delete entry
* is created before we free our unit-flush pending entry, the
* lazy pending entry will be freed AFTER flushed unit's entries.
* we must remove from the lazy list, and go ahead and put into the
* pending delete list, to keep all our dependences together.
* fragment_unlink_for_deletion() will not re-do the unlink for
* the lazy fragment.
* FIXME: this is inefficient b/c no prev ptr in lazy list
*/
add_to_list = remove_from_lazy_deletion_list(dcontext, f);
* a pending delete entry, which must have a lower timestamp
* than ours, so we're all set since vm_area_check_shared_pending()
* now walks all to-be-freed entries in increasing flushtime order.
*/
LOG(THREAD, LOG_CACHE, 5,
"\tlazily-deleted fragment F%d." PFX " removed from lazy list\n", f->id,
f->start_pc);
} else {
LOG(THREAD, LOG_CACHE, 5, "\tadding F%d." PFX " to to-flush list\n", f->id,
f->start_pc);
vm_area_remove_fragment(dcontext, f);
add_to_list = true;
}
if (add_to_list) {
if (prev_f == NULL)
list = f;
else
prev_f->next_vmarea = f;
prev_f = f;
}
pc += FRAG_SIZE(f);
}
ASSERT(pc == unit->cur_pc);
* were moved to pending-delete list, then we'll have no list and
* prev_f==NULL.
*/
if (list == NULL) {
ASSERT(prev_f == NULL);
* or region to flush_fragments_unlink_shared()
*/
STATS_INC(cache_units_flushed_nolive);
} else {
ASSERT(list != NULL);
ASSERT(prev_f != NULL);
prev_f->next_vmarea = NULL;
}
if (tail != NULL)
*tail = prev_f;
return list;
}
* moves those units to the units_to_free list.
* This routine can only be called when !is_self_couldbelinking() and when
* no locks are held.
*/
bool
fcache_flush_pending_units(dcontext_t *dcontext, fragment_t *was_I_flushed)
{
fcache_thread_units_t *tu = (fcache_thread_units_t *)dcontext->fcache_field;
fcache_unit_t *unit_flushed = NULL;
fcache_unit_t *u, *local_to_flush;
bool not_flushed = true;
fragment_t *list_head = NULL, *list_tail = NULL;
fragment_t *list_add_head, *list_add_tail;
uint flushtime;
DEBUG_DECLARE(bool flushed;)
ASSERT(!is_self_couldbelinking());
ASSERT_OWN_NO_LOCKS();
if (!tu->pending_flush)
return not_flushed;
tu->pending_flush = false;
* rather than getting into the flush synch and finding someone beat us
*/
d_r_mutex_lock(&unit_flush_lock);
local_to_flush = allunits->units_to_flush;
allunits->units_to_flush = NULL;
d_r_mutex_unlock(&unit_flush_lock);
if (local_to_flush == NULL)
return not_flushed;
LOG(THREAD, LOG_CACHE, 2, "flushing fragments in all pending units\n");
* pending_flush_units_in_cache, so no lock needed here
*/
if (was_I_flushed != NULL)
unit_flushed = FIFO_UNIT(was_I_flushed);
* manipulating fragments in these units at the same time that we are (e.g.,
* lazily deleting a trace head). Sure, the unit is not on the live
* list anymore, but the fragments are reachable.
*/
DEBUG_DECLARE(flushed =)
flush_fragments_synch_unlink_priv(dcontext, EMPTY_REGION_BASE, EMPTY_REGION_SIZE,
false ,
false ,
false
_IF_DGCDIAG(NULL));
ASSERT(flushed);
KSTART(cache_flush_unit_walk);
for (u = local_to_flush; u != NULL; u = u->next_local) {
ASSERT(u->cache->is_shared);
* use it as a live unit from this point on
*/
DODEBUG({ u->pending_flush = false; });
* on free lists/FIFO empty slots.
* FIXME: would be nice to set this earlier when move to
* units_to_flush list, but then end up w/ freed fragments in the
* middle of the cache that we can't identify in our walk -- so we
* inefficiently put on free list and then take off in the walk.
*/
u->pending_free = true;
if (u == unit_flushed)
not_flushed = false;
list_add_head = chain_fragments_for_flush(dcontext, u, &list_add_tail);
if (list_head == NULL) {
list_head = list_add_head;
list_tail = list_add_tail;
} else if (list_add_head != NULL) {
list_tail->next_vmarea = list_add_head;
list_tail = list_add_tail;
}
ASSERT(list_tail == NULL || list_tail->next_vmarea == NULL);
STATS_INC(cache_units_flushed);
STATS_DEC(cache_units_toflush);
}
KSTOP(cache_flush_unit_walk);
flush_fragments_unlink_shared(dcontext, EMPTY_REGION_BASE, EMPTY_REGION_SIZE,
list_head _IF_DGCDIAG(NULL));
flushtime = flushtime_global;
flush_fragments_end_synch(dcontext, false );
for (u = local_to_flush; u != NULL; u = u->next_local) {
u->flushtime = flushtime;
LOG(THREAD, LOG_CACHE, 2,
"flushed fragments in unit " PFX "-" PFX " @flushtime %u\n", u->start_pc,
u->end_pc, u->flushtime);
}
append_units_to_free_list(local_to_flush);
return not_flushed;
}
void
fcache_free_pending_units(dcontext_t *dcontext, uint flushtime)
{
fcache_unit_t *u, *nxt;
d_r_mutex_lock(&unit_flush_lock);
for (u = allunits->units_to_free; u != NULL; u = nxt) {
nxt = u->next_local;
ASSERT(nxt == NULL || u->flushtime <= nxt->flushtime);
if (u->flushtime <= flushtime) {
if (u == allunits->units_to_free_tail) {
ASSERT(u == allunits->units_to_free);
ASSERT(nxt == NULL);
allunits->units_to_free_tail = NULL;
}
allunits->units_to_free = nxt;
LOG(THREAD, LOG_CACHE, 2, "freeing flushed unit " PFX "-" PFX "\n",
u->start_pc, u->end_pc);
ASSERT(u->pending_free);
u->pending_free = false;
fcache_free_unit(dcontext, u, true);
STATS_INC(cache_units_flushed_freed);
STATS_DEC(cache_units_tofree);
} else
break;
}
d_r_mutex_unlock(&unit_flush_lock);
}
* Caller must be at full synch for a flush.
*/
static void
fcache_mark_units_for_free(dcontext_t *dcontext, fcache_t *cache)
{
fcache_unit_t *u, *head;
ASSERT(is_self_flushing());
PROTECT_CACHE(cache, lock);
* them to free lists.
* Also set flushtime and move them to the pending_free list.
*/
u = cache->units;
ASSERT(u != NULL);
head = u->next_local;
cache->units->next_local = NULL;
for (u = head; u != NULL; u = u->next_local) {
u->pending_free = true;
u->flushtime = flushtime_global;
remove_unit_from_cache(u);
}
PROTECT_CACHE(cache, unlock);
append_units_to_free_list(head);
}
* (while invalidate_code_cache() only flushes all fragments and does not try
* to free any units -- it is meant for consistency purposes, while this is
* meant for capacity purposes).
* FIXME: currently only marks shared cache units for freeing.
* FIXME: should add -stress_flush_units N parameter
*/
void
fcache_flush_all_caches()
{
dcontext_t *dcontext = get_thread_private_dcontext();
ASSERT(dcontext != NULL);
ASSERT_NOT_TESTED();
* FIXME: efficiency of region-based vs unit-based flushing
*/
flush_fragments_in_region_start(
dcontext, UNIVERSAL_REGION_BASE, UNIVERSAL_REGION_SIZE,
false , true ,
false ,
false
_IF_DGCDIAG(NULL));
* so we can mess w/ global cache units in an atomic manner wrt the flush.
* We can't do private here since threads are let go if no shared
* fragments are enabled, but better to have each thread mark its
* own anyway. FIXME -- put flag in delete-list entry
*/
if (DYNAMO_OPTION(shared_bbs)) {
fcache_mark_units_for_free(dcontext, shared_cache_bb);
}
if (DYNAMO_OPTION(shared_traces)) {
fcache_mark_units_for_free(dcontext, shared_cache_trace);
}
* vm_area_flush_fragments() to call a routine here that frees all but
* one unit
*/
flush_fragments_in_region_finish(dcontext, false );
STATS_INC(fcache_flush_all);
}
* over completely, by suspending all other threads and freeing all fragments
* and cache units immediately.
* Can only be called while !couldbelinking.
* Assumes caller holds reset_pending_lock.
* Simultaneous resets are not queued up -- one wins and the rest are canceled.
* Use the schedule_reset() routine to queue up resets of different types, which
* will all be combined.
* FIXME: currently target is ignored and assumed to be RESET_ALL
*/
void
fcache_reset_all_caches_proactively(uint target)
{
thread_record_t **threads;
dcontext_t *my_dcontext = get_thread_private_dcontext();
int i, num_threads;
const thread_synch_state_t desired_state =
THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT_OR_NO_XFER;
* though it can be used to release old tables; but old tables aren't
* needed for hotp_only so they should be stored anyway, which is NYI
* today.
*/
ASSERT(DYNAMO_OPTION(enable_reset) && !RUNNING_WITHOUT_CODE_CACHE());
ASSERT(!is_self_couldbelinking());
ASSERT_OWN_MUTEX(true, &reset_pending_lock);
if (reset_in_progress) {
d_r_mutex_unlock(&reset_pending_lock);
return;
}
* (xref i#3645).
*/
if (!DYNAMO_OPTION(enable_reset)) {
d_r_mutex_unlock(&reset_pending_lock);
return;
}
* we will not be holding some locks we normally would need when
* deleting shared fragments, etc., assuming that we suspend all
* threads in DR while resetting -- if that ever changes we need to
* tighten up all those checks again!
*/
reset_in_progress = true;
* rank it would need to be held across the whole routine
*/
d_r_mutex_unlock(&reset_pending_lock);
LOG(GLOBAL, LOG_CACHE, 2,
"\nfcache_reset_all_caches_proactively: thread " TIDFMT
" suspending all threads\n",
d_r_get_thread_id());
* Case 6821: other synch-all-thread uses can be ignored, as none of them carry
* any non-persistent state.
*/
if (!synch_with_all_threads(desired_state, &threads, &num_threads,
* mcontext->pc to next_tag and use
* THREAD_SYNCH_VALID_MCONTEXT, but some callers (like
* nudge) do not satisfy that */
THREAD_SYNCH_NO_LOCKS_NO_XFER,
* reasons) just abort */
THREAD_SYNCH_SUSPEND_FAILURE_ABORT
* synch issues for whatever reason, bail out sooner
* rather than later */
| THREAD_SYNCH_SMALL_LOOP_MAX)) {
reset_in_progress = false;
ASSERT(!OWN_MUTEX(&all_threads_synch_lock) && !OWN_MUTEX(&thread_initexit_lock));
ASSERT(threads == NULL);
ASSERT(!dynamo_all_threads_synched);
STATS_INC(fcache_reset_abort);
LOG(GLOBAL, LOG_CACHE, 2,
"fcache_reset_all_caches_proactively: aborting due to thread synch "
"failure\n");
* other nudges or sources of thread permission problems */
SYSLOG_INTERNAL_WARNING("proactive reset aborted due to thread synch failure");
return;
}
ASSERT(OWN_MUTEX(&all_threads_synch_lock) && OWN_MUTEX(&thread_initexit_lock));
STATS_INC(fcache_reset_proactively);
DOSTATS({
if (TEST(RESET_PENDING_DELETION, target))
STATS_INC(fcache_reset_pending_del);
});
DOLOG(1, LOG_STATS, {
LOG(GLOBAL, LOG_STATS, 1, "\n**************************Stats BEFORE reset:\n");
dump_global_stats(false);
});
LOG(GLOBAL, LOG_CACHE, 2,
"fcache_reset_all_caches_proactively: walking the threads\n");
char buf[16];
snprintf(buf, BUFFER_SIZE_ELEMENTS(buf), "%d",
GLOBAL_STAT(num_bbs) + GLOBAL_STAT(num_traces));
NULL_TERMINATE_BUFFER(buf);
SYSLOG(SYSLOG_INFORMATION, INFO_RESET_IN_PROGRESS, 3, buf, get_application_name(),
get_application_pid());
* All threads are suspended so no security risk.
*/
SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
dynamo_resetting = true;
IF_AARCHXX({
if (INTERNAL_OPTION(steal_reg_at_reset) != 0)
arch_reset_stolen_reg();
});
* For everything to be freed, it must either be individually freed,
* or it must reside in non-persistent heap units,
* which will be thrown out wholesale in heap_reset_free(). The latter
* is preferable to not waste time on individual deletion.
* XXX: add consistency check walks before and after for all modules
*/
for (i = 0; i < num_threads; i++) {
dcontext_t *dcontext = threads[i]->dcontext;
if (dcontext != NULL) {
LOG(GLOBAL, LOG_CACHE, 2, "\tconsidering thread #%d " TIDFMT "\n", i,
threads[i]->id);
if (dcontext != my_dcontext) {
if (is_thread_currently_native(threads[i])) {
* and lost-control threads are truly native, so no
* state to worry about except for hooks -- and we're
* not freeing the interception buffer.
*/
LOG(GLOBAL, LOG_CACHE, 2,
"\tcurrently native so no translation needed\n");
} else if (thread_synch_state_no_xfer(dcontext)) {
* They have no persistent state, so leave alone.
* Also xref case 7760.
*/
LOG(GLOBAL, LOG_FRAGMENT, 2,
"\tat THREAD_SYNCH_NO_LOCKS_NO_XFER so no translation needed\n");
} else {
translate_from_synchall_to_dispatch(threads[i], desired_state);
}
}
last_exit_deleted(dcontext);
if (target == RESET_PENDING_DELETION) {
* since the reset xfer to d_r_dispatch will disrupt it
*/
if (is_building_trace(dcontext)) {
LOG(THREAD, LOG_FRAGMENT, 2,
"\tsquashing trace of thread " TIDFMT "\n", i);
trace_abort(dcontext);
}
} else {
LOG(GLOBAL, LOG_CACHE, 2, "\tfreeing memory in thread " TIDFMT "\n", i);
LOG(THREAD, LOG_CACHE, 2, "------- reset for thread " TIDFMT " -------\n",
threads[i]->id);
* dcontext owner, esp. wrt tls!
*/
* *_reset() -- can we systematically construct these lists of
* module calls? The list here, though, is a subset of the others.
*/
monitor_thread_reset_free(dcontext);
fragment_thread_reset_free(dcontext);
fcache_thread_reset_free(dcontext);
vm_areas_thread_reset_free(dcontext);
heap_thread_reset_free(dcontext);
}
}
}
if (target == RESET_PENDING_DELETION) {
LOG(GLOBAL, LOG_CACHE, 2,
"fcache_reset_all_caches_proactively: clearing shared deletion list\n");
vm_area_check_shared_pending(GLOBAL_DCONTEXT, NULL);
} else {
fragment_reset_free();
link_reset_free();
fcache_reset_free();
vm_areas_reset_free();
#ifdef HOT_PATCHING_INTERFACE
hotp_reset_free();
#endif
heap_reset_free();
LOG(GLOBAL, LOG_CACHE, 2,
"fcache_reset_all_caches_proactively: re-initializing\n");
heap_reset_init();
#ifdef HOT_PATCHING_INTERFACE
hotp_reset_init();
#endif
vm_areas_reset_init();
fcache_reset_init();
link_reset_init();
fragment_reset_init();
for (i = 0; i < num_threads; i++) {
dcontext_t *dcontext = threads[i]->dcontext;
if (dcontext != NULL) {
LOG(GLOBAL, LOG_CACHE, 2,
"fcache_reset_all_caches_proactively: re-initializing thread " TIDFMT
"\n",
i);
* this before the global free/init since our private/global
* free/init are completely separate (due to the presence of
* persistent state we cannot do a global quick-free anyway).
* But when using shared IBL tables, fragment_reset_init() must
* be called before fragment_thread_reset_init(), since the
* latter copies global state initialized by the former. To
* simplify the code, we simply init all global state prior to
* initing private state (xref case 8092).
*/
heap_thread_reset_init(dcontext);
vm_areas_thread_reset_init(dcontext);
monitor_thread_reset_init(dcontext);
fcache_thread_reset_init(dcontext);
fragment_thread_reset_init(dcontext);
}
}
}
dynamo_resetting = false;
reset_in_progress = false;
SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
DOLOG(1, LOG_STATS, {
LOG(GLOBAL, LOG_STATS, 1, "\n**************************Stats AFTER reset:\n");
dump_global_stats(false);
});
LOG(GLOBAL, LOG_CACHE, 2,
"fcache_reset_all_caches_proactively: resuming all threads\n");
end_synch_with_all_threads(threads, num_threads, true );
}
bool
schedule_reset(uint target)
{
bool added_target;
ASSERT(target != 0);
if (!DYNAMO_OPTION(enable_reset))
return false;
d_r_mutex_lock(&reset_pending_lock);
added_target = !TESTALL(target, reset_pending);
reset_pending |= target;
d_r_mutex_unlock(&reset_pending_lock);
return added_target;
}
#if 0
static void
fcache_reset_cache(dcontext_t *dcontext, fcache_t *cache)
{
fcache_unit_t *u, *prevu;
fragment_t *f;
cache_pc pc, last_pc;
bool unit_empty;
uint num_units = 0;
uint sz;
PROTECT_CACHE(cache, lock);
LOG(THREAD, LOG_CACHE, 2, "fcache_reset_cache %s\n", cache->name);
for (u = cache->units; u != NULL; u = u->next_local) {
LOG(THREAD, LOG_CACHE, 3, " unit %d: "PFX" -> cur "PFX"\n",
num_units, u->start_pc, u->cur_pc);
num_units++;
unit_empty = true;
pc = u->start_pc;
last_pc = pc;
while (pc < u->cur_pc) {
LOG(THREAD, LOG_CACHE, 4, " f @ "PFX"\n", pc);
f = *((fragment_t **)pc);
ASSERT(f != NULL);
ASSERT(FIFO_UNIT(f) == u);
sz = FRAG_SIZE(f);
* Should do an analysis and see if we ever use it anymore.
* It is a powerful feature to support, but also a limiting one...
*/
if (TEST(FRAG_CANNOT_DELETE, f->flags)) {
unit_empty = false;
fifo_prepend_empty(dcontext, cache, u, NULL, last_pc, pc - last_pc);
STATS_FCACHE_SUB(cache, used, (pc - last_pc));
last_pc = pc + sz;
} else {
* with the future fragment that will be created?
* Even worse, what if it triggers a resize of its hashtable?
* Since we're deleting everyone, we should set some flag saying
* "don't create any future fragment for this deleted fragment"
*/
force_fragment_from_cache(dcontext, cache, f);
}
pc += sz;
}
if (last_pc < u->cur_pc) {
fifo_prepend_empty(dcontext, cache, u, NULL, last_pc, pc - last_pc);
STATS_FCACHE_SUB(cache, used, (pc - last_pc));
}
if (unit_empty) {
u->end_pc = NULL;
}
}
for (prevu = NULL, u = cache->units; u != NULL; prevu = u, u = u->next_local) {
if (u->end_pc == NULL) {
u->end_pc = u->start_pc + u->size;
if (num_units > 1) {
num_units--;
fcache_free_unit(dcontext, u, true);
if (prevu == NULL)
u = cache->units;
else
u = prevu->next_local;
}
}
}
* just the tail of the unit?
*/
PROTECT_CACHE(cache, unlock);
}
#endif
* Always safe to free free list (using lock).
* Other allocations must be freed only in the current thread:
* cannot get list of all threads for a global approach b/c that requires memory.
* We let the other threads trip over the low memory trigger to flush their own
* caches.
*/
void
fcache_low_on_memory()
{
fcache_unit_t *u, *next_u;
DEBUG_DECLARE(size_t freed = 0;)
#if 0
dcontext_t *dcontext = get_thread_private_dcontext();
fcache_thread_units_t *tu = (fcache_thread_units_t *) dcontext->fcache_field;
* be called at any alloc point! If in middle of fragment creation,
* we can't just go delete the fragment!
* STRATEGY:
* keep a reserved piece of heap per thread that's big enough to get to
* a safe point from any DR allocation site (perhaps it should use
* a stack allocator). We keep going, using that for memory
* (have to work out shared vs private memory issues if building a
* shared bb), and then at a safe point we reset the cache.
*/
if (tu->bb != NULL)
fcache_reset_cache(dcontext, tu->bb);
if (tu->trace != NULL)
fcache_reset_cache(dcontext, tu->trace);
#endif
LOG(GLOBAL, LOG_CACHE | LOG_STATS, 1,
"fcache_low_on_memory: about to free dead list units\n");
* so we have to be careful what locks we grab.
* No allocation site can hold a lock weaker in rank than heap_unit_lock,
* b/c it could deadlock on the allocation itself! So we only have to worry
* about the locks of rank between heap_alloc_lock and allunits_lock --
* currently dynamo_areas, fcache_unit_areas and global_alloc_lock. We
* check for those locks here. FIXME we have no way to check if holding
* a readlock on the dynamo/fcache_unit_areas lock. FIXME owning the
* dynamo_areas lock here is prob. not that uncommon, we may be able to
* release and re-grab it but would have to be sure that works in all the
* corner cases (if the failing alloc is for a dynamo_areas vector resize
* etc.).
*/
if (lockwise_safe_to_allocate_memory() && !self_owns_dynamo_vm_area_lock() &&
!self_owns_write_lock(&fcache_unit_areas->lock)) {
d_r_mutex_lock(&allunits_lock);
u = allunits->dead;
while (u != NULL) {
next_u = u->next_global;
IF_DEBUG(freed += u->size;)
fcache_really_free_unit(u, true , true );
u = next_u;
}
allunits->dead = NULL;
d_r_mutex_unlock(&allunits_lock);
LOG(GLOBAL, LOG_CACHE | LOG_STATS, 1, "fcache_low_on_memory: freed %d KB\n",
freed / 1024);
} else {
LOG(GLOBAL, LOG_CACHE | LOG_STATS, 1,
"fcache_low_on_memory: cannot walk units b/c of deadlock potential\n");
}
options_make_writable();
* FIXME: I just made this up -- have param to control?
* FIXME: restore params back to original values at some point?
*/
if (dynamo_options.finite_bb_cache && dynamo_options.cache_bb_replace > 0) {
dynamo_options.cache_bb_regen *= 2;
if (dynamo_options.cache_bb_regen > dynamo_options.cache_bb_replace)
dynamo_options.cache_bb_regen = 4 * dynamo_options.cache_bb_replace / 5;
}
if (dynamo_options.finite_shared_bb_cache &&
dynamo_options.cache_shared_bb_replace > 0) {
dynamo_options.cache_shared_bb_regen *= 2;
if (dynamo_options.cache_shared_bb_regen >
dynamo_options.cache_shared_bb_replace) {
dynamo_options.cache_shared_bb_regen =
4 * dynamo_options.cache_shared_bb_replace / 5;
}
}
if (dynamo_options.finite_shared_trace_cache &&
dynamo_options.cache_shared_trace_replace > 0) {
dynamo_options.cache_shared_trace_regen *= 2;
if (dynamo_options.cache_shared_trace_regen >
dynamo_options.cache_shared_trace_replace) {
dynamo_options.cache_shared_trace_regen =
4 * dynamo_options.cache_shared_trace_replace / 5;
}
}
* could get rid of trace cache altogether...
*/
if (dynamo_options.finite_trace_cache && dynamo_options.cache_trace_replace > 0) {
dynamo_options.cache_trace_regen *= 2;
if (dynamo_options.cache_trace_regen > dynamo_options.cache_trace_replace)
dynamo_options.cache_trace_regen = 4 * dynamo_options.cache_trace_replace / 5;
}
options_restore_readonly();
}
* COARSE-GRAIN UNITS
*/
coarse_info_t *
get_fcache_coarse_info(cache_pc pc)
{
fcache_unit_t *unit = fcache_lookup_unit(pc);
if (unit == NULL || unit->cache == NULL)
return NULL;
ASSERT((unit->cache->is_coarse && unit->cache->coarse_info != NULL) ||
(!unit->cache->is_coarse && unit->cache->coarse_info == NULL));
return unit->cache->coarse_info;
}
void
fcache_coarse_cache_delete(dcontext_t *dcontext, coarse_info_t *info)
{
fcache_t *cache;
ASSERT(info != NULL);
ASSERT_OWN_MUTEX(!info->is_local, &info->lock);
cache = (fcache_t *)info->cache;
if (cache == NULL)
return;
* We assume that deletion can only happen for local cache or at reset/exit.
*/
DODEBUG({ cache->is_local = true; });
fcache_cache_free(dcontext, cache, !info->frozen );
info->cache = NULL;
* so nothing else to do here
*/
}
size_t
coarse_frozen_cache_size(dcontext_t *dcontext, coarse_info_t *info)
{
fcache_t *cache;
ASSERT(info != NULL);
ASSERT_OWN_MUTEX(true, &info->lock);
cache = (fcache_t *)info->cache;
if (cache == NULL)
return 0;
* or conversion to 8-bit-jmps
*/
* at end of last unit; we ignore small unused space at end of each unit.
*/
return cache->size - (cache->units->end_pc - cache->units->cur_pc);
}
* and unknown to all but this thread.
*/
void
fcache_coarse_init_frozen(dcontext_t *dcontext, coarse_info_t *info, cache_pc start_pc,
size_t size)
{
fcache_t *cache = fcache_cache_init(GLOBAL_DCONTEXT, FRAG_SHARED | FRAG_COARSE_GRAIN,
false );
* assuming that info is newly created and unknown to all but this thread.
* (For freezing we also have dynamo_all_threads_synched, but we don't for
* loading in persisted caches.)
*/
DODEBUG({ cache->is_local = true; });
cache->units = fcache_create_unit(dcontext, cache, start_pc, size);
DODEBUG({ cache->is_local = false; });
cache->units->cur_pc = cache->units->end_pc;
cache->units->full = true;
cache->coarse_info = info;
info->cache = cache;
}
void
fcache_coarse_set_info(dcontext_t *dcontext, coarse_info_t *info)
{
fcache_t *cache;
ASSERT(info != NULL);
ASSERT_OWN_MUTEX(true, &info->lock);
cache = (fcache_t *)info->cache;
cache->coarse_info = info;
}