* Copyright (c) 2012-2022 Google, Inc. All rights reserved.
* Copyright (c) 2000-2010 VMware, Inc. All rights reserved.
* **********************************************************/
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* * Neither the name of VMware, Inc. nor the names of its contributors may be
* used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
* DAMAGE.
*/
*/
#include "globals.h"
#include "fragment.h"
#include "link.h"
#include "utils.h"
#include "emit.h"
#include "fcache.h"
#include "monitor.h"
#include "instrument.h"
#include "instr.h"
#include "perscache.h"
#include "disassemble.h"
* make monitor_data_t opaque in globals.h.
*/
extern bool
mangle_trace(dcontext_t *dcontext, instrlist_t *ilist, monitor_data_t *md);
* so we should make the buffer size bigger, if a client is used.*/
#define MAX_TRACE_BUFFER_SIZE MAX_FRAGMENT_SIZE
* for x64, the rip-rel instrs prevent a memcpy on a resize
*/
#ifdef X64
# define INITIAL_TRACE_BUFFER_SIZE MAX_TRACE_BUFFER_SIZE
#else
# define INITIAL_TRACE_BUFFER_SIZE 1024
#endif
#define INITIAL_NUM_BLKS 8
#define INIT_COUNTER_TABLE_SIZE 9
#define COUNTER_TABLE_LOAD 75
* we don't support local unprotected so we use global
*/
#define COUNTER_ALLOC(dc, ...) \
(TEST(SELFPROT_LOCAL, dynamo_options.protect_mask) \
? global_unprotected_heap_alloc(__VA_ARGS__) \
: heap_alloc(dc, __VA_ARGS__))
#define COUNTER_FREE(dc, p, ...) \
(TEST(SELFPROT_LOCAL, dynamo_options.protect_mask) \
? global_unprotected_heap_free(p, __VA_ARGS__) \
: heap_free(dc, p, __VA_ARGS__))
static void
reset_trace_state(dcontext_t *dcontext, bool grab_link_lock);
DECLARE_CXTSWPROT_VAR(mutex_t trace_building_lock, INIT_LOCK_FREE(trace_building_lock));
* using a sentinel value to determine whether we've built a trace or not
*/
#define TH_COUNTER_CREATED_TRACE_VALUE() (INTERNAL_OPTION(trace_threshold) + 1U)
static void
delete_private_copy(dcontext_t *dcontext)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
if (md->last_copy != NULL) {
LOG(THREAD, LOG_MONITOR, 4, "Deleting previous private copy F%d (" PFX ")\n",
md->last_copy->id, md->last_copy->tag);
* if last_copy is last_fragment
*/
if (md->last_copy == md->last_fragment) {
md->last_fragment = NULL;
}
if (md->last_copy == dcontext->last_fragment)
last_exit_deleted(dcontext);
if (TEST(FRAG_WAS_DELETED, md->last_copy->flags)) {
* needed for pc translation (at least until -safe_translate_flushed
* is on by default), so if we come here later we must check
* for an intervening flush to avoid double-deletion.
*/
LOG(THREAD, LOG_MONITOR, 4,
"\tprivate copy was flushed so not re-deleting\n");
STATS_INC(num_trace_private_deletions_flushed);
} else {
fragment_delete(dcontext, md->last_copy,
FRAGDEL_NO_MONITOR
| FRAGDEL_NO_HTABLE);
}
md->last_copy = NULL;
STATS_INC(num_trace_private_deletions);
}
}
static void
create_private_copy(dcontext_t *dcontext, fragment_t *f)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
* violates instrlist_t abstraction, have to deal w/ changes for
* bb->trace (like ibl target) and worry about encoding process
* changing instr_t state in a way that will affect the trace...
*
* instead we re-decode the thing, for simplicity
*/
KSTART(temp_private_bb);
LOG(THREAD, LOG_MONITOR, 4,
"Creating private copy of F%d (" PFX ") for trace creation\n", f->id, f->tag);
ASSERT(dr_get_isa_mode(dcontext) ==
FRAG_ISA_MODE(f->flags)
IF_X86_64(||
(dr_get_isa_mode(dcontext) == DR_ISA_IA32 &&
!FRAG_IS_32(f->flags) && DYNAMO_OPTION(x86_to_x64))));
* we delete here, when we add a new copy, and not in internal_restore_last
* since if we do it there we'll clobber last_exit too early
*/
if (md->last_copy != NULL)
delete_private_copy(dcontext);
* frozen coarse fragments, we re-build from app code (which also simplifies
* our client trace model). If the existing f was flushed/deleted, that won't
* stop us from creating a new one for our trace.
*/
* unfortunately -- need to be transactional so we finish building this guy,
* and then just stop (will delete on next trace build)
*/
md->last_fragment = build_basic_block_fragment(
dcontext, f->tag, FRAG_TEMP_PRIVATE, true ,
* thread-private versions already exist, so
* we have to make them invisible */
false, true , md->pass_to_client ? &md->unmangled_bb_ilist : NULL);
md->last_copy = md->last_fragment;
STATS_INC(num_trace_private_copies);
LOG(THREAD, LOG_MONITOR, 4,
"Created private copy F%d of original F%d (" PFX ") for trace creation\n",
md->last_fragment->id, f->id, f->tag);
DOLOG(2, LOG_INTERP, {
disassemble_fragment(dcontext, md->last_fragment, d_r_stats->loglevel <= 3);
});
KSTOP(temp_private_bb);
ASSERT(!TEST(FRAG_CANNOT_BE_TRACE, md->last_fragment->flags));
}
static void
extend_unmangled_ilist(dcontext_t *dcontext, fragment_t *f)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
if (md->pass_to_client) {
instr_t *inst;
* of walking exit stubs here?
* FIXME: remove once we have PR 307284.
*/
linkstub_t *l;
ASSERT(md->last_copy != NULL);
ASSERT(!TEST(FRAG_COARSE_GRAIN, md->last_copy->flags));
for (l = FRAGMENT_EXIT_STUBS(md->last_copy); LINKSTUB_NEXT_EXIT(l) != NULL;
l = LINKSTUB_NEXT_EXIT(l))
;
md->final_exit_flags = l->flags;
LOG(THREAD, LOG_MONITOR, 2, "final exit flags: %x\n", md->final_exit_flags);
ASSERT(md->unmangled_bb_ilist != NULL);
if (instrlist_first(md->unmangled_bb_ilist) != NULL) {
instrlist_append(&md->unmangled_ilist,
instrlist_first(md->unmangled_bb_ilist));
}
DOLOG(4, LOG_INTERP, {
LOG(THREAD, LOG_INTERP, 4, "unmangled ilist with F%d(" PFX "):\n",
md->last_copy->id, md->last_copy->tag);
instrlist_disassemble(dcontext, md->trace_tag, &md->unmangled_ilist, THREAD);
});
ASSERT(md->num_blks < md->blk_info_length);
inst = instrlist_last(md->unmangled_bb_ilist);
md->blk_info[md->num_blks].vmlist = NULL;
if (inst != NULL) {
vm_area_add_to_list(dcontext, f->tag, &(md->blk_info[md->num_blks].vmlist),
md->trace_flags, f, false );
md->blk_info[md->num_blks].final_cti =
instr_is_cti(instrlist_last(md->unmangled_bb_ilist));
} else
md->blk_info[md->num_blks].final_cti = false;
instrlist_init(md->unmangled_bb_ilist);
instrlist_destroy(dcontext, md->unmangled_bb_ilist);
md->unmangled_bb_ilist = NULL;
}
* then store for the trace.
*/
if (md->last_copy != NULL && TEST(FRAG_HAS_TRANSLATION_INFO, md->last_copy->flags))
md->trace_flags |= FRAG_HAS_TRANSLATION_INFO;
}
bool
mangle_trace_at_end(void)
{
* unless there's a client bb or trace hook, for a for-cache trace
* or a recreate-state trace.
*/
return (dr_bb_hook_exists() || dr_trace_hook_exists());
}
void
d_r_monitor_init()
{
* so we have to stop a trace at that size
* this does not include exit stubs
*/
ASSERT(MAX_TRACE_BUFFER_SIZE <= MAX_FRAGMENT_SIZE);
}
void
monitor_thread_reset_init(dcontext_t *dcontext)
{
}
void
monitor_thread_reset_free(dcontext_t *dcontext)
{
trace_abort_and_delete(dcontext);
}
void
trace_abort_and_delete(dcontext_t *dcontext)
{
trace_abort(dcontext);
* removed in trace_abort (at least until -safe_translate_flushed is on)
*/
delete_private_copy(dcontext);
}
void
d_r_monitor_exit()
{
LOG(GLOBAL, LOG_MONITOR | LOG_STATS, 1, "Trace fragments generated: %d\n",
GLOBAL_STAT(num_traces));
DELETE_LOCK(trace_building_lock);
}
static void
thcounter_free(dcontext_t *dcontext, void *p)
{
COUNTER_FREE(dcontext, p, sizeof(trace_head_counter_t) HEAPACCT(ACCT_THCOUNTER));
}
void
monitor_thread_init(dcontext_t *dcontext)
{
monitor_data_t *md;
md = (monitor_data_t *)heap_alloc(dcontext,
sizeof(monitor_data_t) HEAPACCT(ACCT_TRACE));
dcontext->monitor_field = (void *)md;
memset(md, 0, sizeof(monitor_data_t));
reset_trace_state(dcontext, false );
* FIXME: could set initial sizes to 0 for all configurations, instead
* FIXME: we can optimize even more to not allocate md at all, but would need
* to have hotp_only checks in monitor_cache_exit(), etc.
*/
if (RUNNING_WITHOUT_CODE_CACHE() || DYNAMO_OPTION(disable_traces))
return;
md->thead_table = generic_hash_create(
dcontext, INIT_COUNTER_TABLE_SIZE, COUNTER_TABLE_LOAD,
* traces and trace-building efficiency
*/
HASHTABLE_PERSISTENT, thcounter_free _IF_DEBUG("trace heads"));
md->thead_table->hash_func = HASH_FUNCTION_MULTIPLY_PHI;
}
void
monitor_thread_exit(dcontext_t *dcontext)
{
DEBUG_DECLARE(monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;)
* We call trace_abort so that in case the thread is terminated in
* the middle of trace building from a shared trace head, it has a
* chance to clear the FRAG_TRACE_BUILDING flag. Otherwise, a trace
* can never be built from that particular trace head.
*/
trace_abort(dcontext);
#ifdef DEBUG
if (md->trace_buf != NULL) {
heap_reachable_free(dcontext, md->trace_buf,
md->trace_buf_size HEAPACCT(ACCT_TRACE));
}
if (md->blk_info != NULL) {
heap_free(dcontext, md->blk_info,
md->blk_info_length * sizeof(trace_bb_build_t) HEAPACCT(ACCT_TRACE));
}
if (md->thead_table != NULL)
generic_hash_destroy(dcontext, md->thead_table);
heap_free(dcontext, md, sizeof(monitor_data_t) HEAPACCT(ACCT_TRACE));
#endif
}
static trace_head_counter_t *
thcounter_lookup(dcontext_t *dcontext, app_pc tag)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
return (trace_head_counter_t *)generic_hash_lookup(dcontext, md->thead_table,
(ptr_uint_t)tag);
}
static trace_head_counter_t *
thcounter_add(dcontext_t *dcontext, app_pc tag)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
trace_head_counter_t *e = thcounter_lookup(dcontext, tag);
if (e == NULL) {
e = COUNTER_ALLOC(dcontext,
sizeof(trace_head_counter_t) HEAPACCT(ACCT_THCOUNTER));
e->tag = tag;
e->counter = 0;
generic_hash_add(dcontext, md->thead_table, (ptr_uint_t)tag, e);
}
return e;
}
void
thcounter_range_remove(dcontext_t *dcontext, app_pc start, app_pc end)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
if (md->thead_table != NULL) {
generic_hash_range_remove(dcontext, md->thead_table, (ptr_uint_t)start,
(ptr_uint_t)end);
}
}
bool
is_building_trace(dcontext_t *dcontext)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
return (md->trace_tag != NULL);
}
app_pc
cur_trace_tag(dcontext_t *dcontext)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
return md->trace_tag;
}
void *
cur_trace_vmlist(dcontext_t *dcontext)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
return md->trace_vmlist;
}
static void
reset_trace_state(dcontext_t *dcontext, bool grab_link_lock)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
uint i;
instrlist_init(&(md->trace));
if (instrlist_first(&md->unmangled_ilist) != NULL)
instrlist_clear(dcontext, &md->unmangled_ilist);
instrlist_init(&md->unmangled_ilist);
if (md->unmangled_bb_ilist != NULL)
instrlist_clear_and_destroy(dcontext, md->unmangled_bb_ilist);
md->unmangled_bb_ilist = NULL;
md->trace_buf_top = 0;
ASSERT(md->trace_vmlist == NULL);
for (i = 0; i < md->num_blks; i++) {
vm_area_destroy_list(dcontext, md->blk_info[i].vmlist);
md->blk_info[i].vmlist = NULL;
}
md->num_blks = 0;
* FRAG_TRACE_BUILDING set on a shared BB w/the same tag (if there is a
* BB present -- it could've been deleted for cache management or cache
* consistency). Unset the flag so that a trace can be built from it
* in the future.
*/
if (TEST(FRAG_SHARED, md->trace_flags) && DYNAMO_OPTION(shared_bbs)) {
* if a shared BB is present. */
fragment_t *bb = fragment_lookup_shared_bb(dcontext, md->trace_tag);
* we can't expect it to be set simply because the BB is shared. Check
* just for the trace bulding flag.
*/
if (grab_link_lock)
acquire_recursive_lock(&change_linking_lock);
if (bb != NULL && TEST(FRAG_TRACE_BUILDING, bb->flags)) {
* flag. The regenerated fragment could have another thread building
* a trace from it so the clear would be for the the wrong thread
* here. It doesn't cause a correctness problem because the
* emit-time race detection logic will catch it. (In testing w/IIS,
* we've seen very, very few emit-time aborts -- < 1% of all races.)
*/
ASSERT(TESTALL(FRAG_SHARED | FRAG_IS_TRACE_HEAD, bb->flags));
STATS_INC(num_trace_building_ip_cleared);
bb->flags &= ~FRAG_TRACE_BUILDING;
}
#ifdef DEBUG
* should be rare in most apps but we'll track it w/a counter in case
* we see lots of emit-time aborts.
*/
else {
STATS_INC(num_reset_trace_no_trace_head);
* re-genned and so wouldn't be marked as FRAG_TRACE_BUILDING. It might
* be marked as a trace head, though, so we don't assert anything about
* that trait.
* FIXME We could add a strong ASSERT about the regen case if we added
* a trace_head_id field to monitor_data_t. The field would store the id
* of the shared BB trace head that caused trace building to begin. If
* a shared trace head isn't found but a shared BB is, the shared BB
* id should be greater than trace_head_id.
*/
}
#endif
if (grab_link_lock)
release_recursive_lock(&change_linking_lock);
}
md->trace_tag = NULL;
md->trace_flags = 0;
md->emitted_size = 0;
ASSERT(md->last_fragment == NULL ||
(md->last_fragment_flags & (FRAG_CANNOT_DELETE | FRAG_LINKED_OUTGOING)) ==
(md->last_fragment->flags & (FRAG_CANNOT_DELETE | FRAG_LINKED_OUTGOING)));
md->last_fragment_flags = 0;
* a fragment we're examining (seg fault, etc.)
*/
md->last_fragment = NULL;
* (at least until -safe_translate_flushed is on) (xref case 8083)
*/
#ifdef CUSTOM_TRACES_RET_REMOVAL
dcontext->call_depth = 0;
#endif
}
bool
monitor_delete_would_abort_trace(dcontext_t *dcontext, fragment_t *f)
{
monitor_data_t *md;
if (dcontext == GLOBAL_DCONTEXT)
dcontext = get_thread_private_dcontext();
if (dcontext == NULL)
return false;
md = (monitor_data_t *)dcontext->monitor_field;
return ((md->last_fragment == f || dcontext->last_fragment == f) &&
md->trace_tag != NULL);
}
void
monitor_remove_fragment(dcontext_t *dcontext, fragment_t *f)
{
monitor_data_t *md;
if (dcontext == GLOBAL_DCONTEXT) {
ASSERT(TEST(FRAG_SHARED, f->flags));
dcontext = get_thread_private_dcontext();
if (dcontext == NULL) {
if (dynamo_exited)
return;
ASSERT_NOT_REACHED();
return;
}
}
md = (monitor_data_t *)dcontext->monitor_field;
if (md->last_copy == f) {
md->last_copy = NULL;
STATS_INC(num_trace_private_deletions);
}
* trace, is being deleted before we're done with it.
* This can happen due to a flush from self-modifying code,
* or an munmap.
* Must check both last_fragment and last_exit.
* May come here before last_exit is set, or may come here after
* last_fragment is restored but before last_exit is used.
* FIXME: if we do manage to remove the check for last_fragment
* here, remove the last_exit clear in end_and_emit_trace
*/
* starts at the next_tag and last_fragment is really not
* related.
*/
if ((md->last_fragment == f || dcontext->last_fragment == f) &&
!TEST(FRAG_TEMP_PRIVATE, f->flags)) {
if (md->trace_tag != NULL) {
LOG(THREAD, LOG_MONITOR, 2, "Aborting current trace since F%d was deleted\n",
f->id);
trace_abort(dcontext);
}
* mode, it should not be set!
*/
ASSERT(md->last_fragment == NULL);
if (dcontext->last_fragment == f)
last_exit_deleted(dcontext);
}
}
static inline void
unlink_ibt_trace_head(dcontext_t *dcontext, fragment_t *f)
{
ASSERT(TEST(FRAG_IS_TRACE_HEAD, f->flags));
if (DYNAMO_OPTION(shared_bb_ibt_tables)) {
ASSERT(TEST(FRAG_SHARED, f->flags));
if (fragment_prepare_for_removal(GLOBAL_DCONTEXT, f)) {
LOG(THREAD, LOG_FRAGMENT, 3, " F%d(" PFX ") removed as trace head IBT\n",
f->id, f->tag);
STATS_INC(num_th_bb_ibt_unlinked);
}
} else {
* property, we must unlink the fragment from every thread's IBT tables.
* This is a heavyweight operation compared to the use of a shared table
* and requires additional changes -- for example, get_list_of_threads()
* can't currently be called from here. If we change trace head-ness
* to a private property, this becomes very easy and more performant
* than the use of a shared table. (Case 3530 discusses private vs shared
* trace head-ness.)
*/
thread_record_t **threads;
int num_threads;
int i;
ASSERT_NOT_IMPLEMENTED(false);
* remove completely here */
fragment_remove_from_ibt_tables(dcontext, f, false );
d_r_mutex_lock(&thread_initexit_lock);
get_list_of_threads(&threads, &num_threads);
d_r_mutex_unlock(&thread_initexit_lock);
for (i = 0; i < num_threads; i++) {
dcontext_t *tgt_dcontext = threads[i]->dcontext;
LOG(THREAD, LOG_FRAGMENT, 2, " considering thread %d/%d = " TIDFMT "\n",
i + 1, num_threads, threads[i]->id);
ASSERT(is_thread_known(tgt_dcontext->owning_thread));
fragment_prepare_for_removal(
TEST(FRAG_SHARED, f->flags) ? GLOBAL_DCONTEXT : tgt_dcontext, f);
}
global_heap_free(
threads, num_threads * sizeof(thread_record_t *) HEAPACCT(ACCT_THREAD_MGT));
}
}
void
mark_trace_head(dcontext_t *dcontext_in, fragment_t *f, fragment_t *src_f,
linkstub_t *src_l)
{
bool protected = false;
cache_pc coarse_stub = NULL, coarse_body = NULL;
coarse_info_t *info = NULL;
* removing a fine trace that triggers a shift to its shadowed coarse trace
* head and a link_fragment_incoming on that head.
* Using the flushing thread's dcontext for the trace head counter is fine
* and shouldn't limit its becoming a new trace again.
*/
dcontext_t *dcontext =
(dcontext_in == GLOBAL_DCONTEXT) ? get_thread_private_dcontext() : dcontext_in;
ASSERT(dcontext != NULL);
LOG(THREAD, LOG_MONITOR, 3, "marking F%d (" PFX ") as trace head\n", f->id, f->tag);
ASSERT(!TEST(FRAG_IS_TRACE, f->flags));
ASSERT(!NEED_SHARED_LOCK(f->flags) || self_owns_recursive_lock(&change_linking_lock));
if (thcounter_lookup(dcontext, f->tag) == NULL) {
protected = local_heap_protected(dcontext);
if (protected) {
protect_local_heap(dcontext, WRITABLE);
}
* counter field is not in fragment_t...
* Move counters to Future for all uses, giving us persistent counters too!
*/
thcounter_add(dcontext, f->tag);
} else {
STATS_INC(trace_head_remark);
}
LOG(THREAD, LOG_MONITOR, 4, "mark_trace_head: flags 0x%08x\n", f->flags);
f->flags |= FRAG_IS_TRACE_HEAD;
LOG(THREAD, LOG_MONITOR, 4, "\tnow, flags 0x%08x\n", f->flags);
LOG(THREAD, LOG_MONITOR, 4, "unlinking incoming for new trace head F%d (" PFX ")\n",
f->id, f->tag);
if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
* (more specifically, on the entrance stub taken). If we don't have
* any info on src_f we use f's unit.
*/
info = get_fragment_coarse_info(src_f == NULL ? f : src_f);
if (info == NULL) {
* so there is nothing to unlink.
*/
} else {
fragment_coarse_lookup_in_unit(dcontext, info, f->tag, &coarse_stub,
&coarse_body);
* shouldn't happen, except perhaps with clients.
*/
ASSERT(src_f != NULL || !info->frozen);
if (src_f != NULL && TEST(FRAG_COARSE_GRAIN, src_f->flags) && src_l != NULL &&
LINKSTUB_NORMAL_DIRECT(src_l->flags)) {
direct_linkstub_t *dl = (direct_linkstub_t *)src_l;
if (dl->stub_pc != NULL && coarse_is_entrance_stub(dl->stub_pc)) {
if (coarse_stub == NULL) {
* is another unit and does not yet have an entrance
* stub in the new fragment's unit, we will come here
* w/o that entrance stub being in the htable. We rely
* on dl->stub_pc being set to that entrance stub.
*/
coarse_stub = dl->stub_pc;
} else
ASSERT(dl->stub_pc == NULL || dl->stub_pc == coarse_stub);
}
}
if (coarse_stub != NULL) {
ASSERT(coarse_is_entrance_stub(coarse_stub));
* (in particular, fragment_coarse_link_wrapper() calling
* fragment_coarse_lookup_wrapper() does not), and we
* un-mark as trace heads when linking incoming (case 8907),
* so we may get here for an existing trace head.
*/
if (!coarse_is_trace_head_in_own_unit(dcontext, f->tag, coarse_stub,
coarse_body, true,
(src_f == NULL) ? info : NULL)) {
ASSERT(coarse_body == NULL ||
entrance_stub_jmp_target(coarse_stub) == coarse_body);
if (coarse_body == NULL &&
(src_f == NULL || get_fragment_coarse_info(f) == info)) {
coarse_body = FCACHE_ENTRY_PC(f);
}
coarse_mark_trace_head(dcontext, f, info, coarse_stub, coarse_body);
}
} else {
LOG(THREAD, LOG_MONITOR, 4, "\tno local stub, deferring th unlink\n");
* stub will be unlinked and its body pc added to the th table in
* link_new_coarse_grain_fragment(); or the source is a fine
* fragment corresponding to another unit and thus no entrance stub
* or htable changes are necessary.
*/
STATS_INC(coarse_th_from_fine);
ASSERT_CURIOSITY(
GLOBAL_STAT(num_fragments) == f->id ||
(src_f != NULL && !TEST(FRAG_COARSE_GRAIN, src_f->flags)));
}
}
} else
unlink_fragment_incoming(dcontext, f);
if (DYNAMO_OPTION(bb_ibl_targets))
unlink_ibt_trace_head(dcontext, f);
#ifdef TRACE_HEAD_CACHE_INCR
* re-link), since combined they aren't atomic, separate atomic
* steps w/ ok intermediate (go back to DR) is fine
*/
* FIXME: we get called in the middle of linking new fragments, so
* we end up linking some incoming links twice (no harm done except
* a waste of time) -- how fix it?
* When fix it, change link_branch to assert that !already linked
*/
link_fragment_incoming(dcontext, f, false );
#endif
STATS_INC(num_trace_heads_marked);
* re-protecting fcache
*/
if (protected) {
protect_local_heap(dcontext, READONLY);
}
}
* to avoid recursion when re-verifying with change_linking_lock held
*/
static bool
should_be_trace_head_internal_unsafe(dcontext_t *dcontext, fragment_t *from_f,
linkstub_t *from_l, app_pc to_tag, uint to_flags,
bool trace_sysenter_exit)
{
app_pc from_tag;
uint from_flags;
if (DYNAMO_OPTION(disable_traces) || TEST(FRAG_IS_TRACE, to_flags) ||
TEST(FRAG_IS_TRACE_HEAD, to_flags) || TEST(FRAG_CANNOT_BE_TRACE, to_flags))
return false;
if (trace_sysenter_exit)
return true;
from_tag = from_f->tag;
from_flags = from_f->flags;
* 1) a link from a trace, or
* 2) a backward direct branch
* Watch out -- since we stop building traces at trace heads,
* too many can hurt performance, especially if bbs do not follow
* direct ctis. We can use shadowed bbs to go through trace
* head and trace boundaries for custom traces.
*/
if (TEST(FRAG_IS_TRACE, from_flags) ||
(to_tag <= from_tag && LINKSTUB_DIRECT(from_l->flags)))
return true;
DOSTATS({
if (!DYNAMO_OPTION(disable_traces) && !TEST(FRAG_IS_TRACE, to_flags) &&
!TEST(FRAG_IS_TRACE_HEAD, to_flags) &&
!TEST(FRAG_CANNOT_BE_TRACE, to_flags)) {
STATS_INC(num_wannabe_traces);
}
});
return false;
}
* trace head based on fragment traits and/or control flow between the
* link stub and the to_tag/to_flags.
*
* For -shared_bbs, will return TRACE_HEAD_OBTAINED_LOCK if the
* change_linking_lock is not already held (meaning from_l->fragment is
* private) and the to_tag is FRAG_SHARED and TRACE_HEAD_YES is being returned,
* since the change_linking_lock must be held and the TRACE_HEAD_YES result
* re-verified. In that case the caller must free the change_linking_lock.
* If trace_sysenter_exit = true, control flow rules are not checked, i.e., the
* from_l and to_tag params are not checked. This is provided to capture
* the case where the most recent cache exit was prior to a non-ignorable
* syscall via a SYSENTER instruction. See comments in monitor_cache_exit for
* details. This is the exception, not the norm.
*
* If the link stub is non-NULL, trace_sysenter_exit does NOT need to
* be set.
*
* FIXME This is a stopgap soln. The long-term fix is to not count on
* a link stub being passed in but rather pass in the most recent fragment's
* flags & tag explicitly. The flags & tag can be stored in a dcontext-private
* monitor structure, one that is not shared across call backs.
*/
static uint
should_be_trace_head_internal(dcontext_t *dcontext, fragment_t *from_f,
linkstub_t *from_l, app_pc to_tag, uint to_flags,
bool have_link_lock, bool trace_sysenter_exit)
{
uint result = 0;
if (should_be_trace_head_internal_unsafe(dcontext, from_f, from_l, to_tag, to_flags,
trace_sysenter_exit)) {
result |= TRACE_HEAD_YES;
ASSERT(!have_link_lock || self_owns_recursive_lock(&change_linking_lock));
if (!have_link_lock) {
* re-verify that it hasn't already been marked.
* If source is also shared then lock should already be held .
*/
ASSERT(from_l == NULL || !NEED_SHARED_LOCK(from_f->flags));
if (NEED_SHARED_LOCK(to_flags)) {
acquire_recursive_lock(&change_linking_lock);
if (should_be_trace_head_internal_unsafe(dcontext, from_f, from_l, to_tag,
to_flags, trace_sysenter_exit)) {
result |= TRACE_HEAD_OBTAINED_LOCK;
} else {
result &= ~TRACE_HEAD_YES;
release_recursive_lock(&change_linking_lock);
}
}
}
}
return result;
}
* trace head based on fragment traits and/or control flow between the
* link stub and the to_tag/to_flags.
*
* For -shared_bbs, will return TRACE_HEAD_OBTAINED_LOCK if the
* change_linking_lock is not already held (meaning from_l->fragment is
* private) and the to_tag is FRAG_SHARED and TRACE_HEAD_YES is being returned,
* since the change_linking_lock must be held and the TRACE_HEAD_YES result
* re-verified. In that case the caller must free the change_linking_lock.
*/
uint
should_be_trace_head(dcontext_t *dcontext, fragment_t *from_f, linkstub_t *from_l,
app_pc to_tag, uint to_flags, bool have_link_lock)
{
return should_be_trace_head_internal(dcontext, from_f, from_l, to_tag, to_flags,
have_link_lock, false);
}
*/
static bool
check_for_trace_head(dcontext_t *dcontext, fragment_t *from_f, linkstub_t *from_l,
fragment_t *to_f, bool have_link_lock, bool trace_sysenter_exit)
{
if (!DYNAMO_OPTION(disable_traces)) {
uint th = should_be_trace_head_internal(dcontext, from_f, from_l, to_f->tag,
to_f->flags, have_link_lock,
trace_sysenter_exit);
if (TEST(TRACE_HEAD_YES, th)) {
mark_trace_head(dcontext, to_f, from_f, from_l);
if (TEST(TRACE_HEAD_OBTAINED_LOCK, th))
release_recursive_lock(&change_linking_lock);
return true;
}
}
return false;
}
* This routines also marks new traces heads if mark_new_trace_head is true.
* The current implementation of this routine assumes that we don't
* want to link potential trace heads. A potential trace head is any
* block fragment that is reached by a backward (direct) branch.
*/
bool
monitor_is_linkable(dcontext_t *dcontext, fragment_t *from_f, linkstub_t *from_l,
fragment_t *to_f, bool have_link_lock, bool mark_new_trace_head)
{
if (TEST(FRAG_IS_TRACE, from_f->flags) && TEST(FRAG_IS_TRACE, to_f->flags))
return true;
if (DYNAMO_OPTION(disable_traces))
return true;
#ifndef TRACE_HEAD_CACHE_INCR
if (TEST(FRAG_IS_TRACE_HEAD, to_f->flags) && !DYNAMO_OPTION(disable_traces))
return false;
#endif
if (mark_new_trace_head) {
uint th = should_be_trace_head(dcontext, from_f, from_l, to_f->tag, to_f->flags,
have_link_lock);
if (TEST(TRACE_HEAD_YES, th)) {
mark_trace_head(dcontext, to_f, from_f, from_l);
if (TEST(TRACE_HEAD_OBTAINED_LOCK, th))
release_recursive_lock(&change_linking_lock);
#ifdef TRACE_HEAD_CACHE_INCR
* link will end up pointing not to fcache_return but to trace_head_incr
*/
return true;
#else
return false;
#endif
}
}
return true;
}
* hold add_size more bytes.
* If the resulting size will exceed the maximum trace
* buffer size, returns false, else returns true.
* FIXME: now that we have a real max limit on emitted trace size,
* should we have an unbounded trace buffer size?
* Also increases the size of the block array if necessary.
*/
static bool
make_room_in_trace_buffer(dcontext_t *dcontext, uint add_size, fragment_t *f)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
uint size;
uint new_blks;
ssize_t realloc_shift;
instr_t *instr;
instrlist_t *trace = &md->trace;
size = md->trace_buf_size;
if (add_size > (size - md->trace_buf_top)) {
byte *new_tbuf;
if (size == 0)
size = INITIAL_TRACE_BUFFER_SIZE;
while (add_size > (size - md->trace_buf_top))
size *= 2;
if (size > MAX_TRACE_BUFFER_SIZE) {
LOG(THREAD, LOG_MONITOR, 2, "Not letting trace buffer grow to %d bytes\n",
size);
return false;
}
new_tbuf = heap_reachable_alloc(dcontext, size HEAPACCT(ACCT_TRACE));
if (md->trace_buf != NULL) {
IF_X64(ASSERT_NOT_REACHED());
memcpy(new_tbuf, md->trace_buf, md->trace_buf_size);
heap_reachable_free(dcontext, md->trace_buf,
md->trace_buf_size HEAPACCT(ACCT_TRACE));
realloc_shift = new_tbuf - md->trace_buf;
instr = instrlist_first(trace);
while (instr != NULL) {
byte *b = instr_get_raw_bits(instr);
if (b >= md->trace_buf && b < md->trace_buf + md->trace_buf_size)
instr_shift_raw_bits(instr, realloc_shift);
instr = instr_get_next(instr);
}
}
LOG(THREAD, LOG_MONITOR, 3,
"\nRe-allocated trace buffer from %d @" PFX " to %d bytes @" PFX "\n",
md->trace_buf_size, md->trace_buf, size, new_tbuf);
md->trace_buf = new_tbuf;
md->trace_buf_size = size;
}
if ((f->flags & FRAG_IS_TRACE) != 0) {
trace_only_t *t = TRACE_FIELDS(f);
new_blks = t->num_bbs;
} else
new_blks = 1;
if (md->num_blks + new_blks >= md->blk_info_length) {
trace_bb_build_t *new_buf;
uint new_len = md->blk_info_length;
if (new_len == 0)
new_len = INITIAL_NUM_BLKS;
do {
new_len *= 2;
} while (md->num_blks + new_blks >= new_len);
new_buf = (trace_bb_build_t *)HEAP_ARRAY_ALLOC(dcontext, trace_bb_build_t,
new_len, ACCT_TRACE, true);
memset(new_buf, 0, sizeof(trace_bb_build_t) * new_len);
LOG(THREAD, LOG_MONITOR, 3, "\nRe-allocating trace blks from %d to %d\n",
md->blk_info_length, new_len);
if (md->blk_info != NULL) {
memcpy(new_buf, md->blk_info, md->blk_info_length * sizeof(trace_bb_build_t));
HEAP_ARRAY_FREE(dcontext, md->blk_info, trace_bb_build_t, md->blk_info_length,
ACCT_TRACE, true);
}
md->blk_info = new_buf;
md->blk_info_length = new_len;
}
return true;
}
static int
trace_exit_stub_size_diff(dcontext_t *dcontext, fragment_t *f)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
int size = 0;
linkstub_t *l;
for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
if (linkstub_shares_next_stub(dcontext, f, l)) {
* share (if client adds custom code, etc.)
* this also makes fixup_last_cti() code simpler since it can
* blindly remove and ignore sharing.
* if the trace does share for a final bb, we remove in
* end_and_emit_trace().
*/
size += local_exit_stub_size(dcontext, EXIT_TARGET_TAG(dcontext, f, l),
md->trace_flags);
} else {
* the difference here, not the absolute new size
*/
size += local_exit_stub_size(dcontext, EXIT_TARGET_TAG(dcontext, f, l),
md->trace_flags) -
local_exit_stub_size(dcontext, EXIT_TARGET_TAG(dcontext, f, l), f->flags);
}
}
return size;
}
enum { MAX_TRACE_FRACTION_OF_CACHE = 8 };
* to be added to it.
* If that size exceeds the maximum fragment size, or a fraction of the maximum
* trace cache size, returns false.
* Returns the size calculations in two different parts:
* res_add_size is the accurate value of the body and exit stubs addition, while
* res_prev_mangle_size is an upper bound estimate of the change in size when
* the prior block in the trace is mangled to connect to f.
*/
static bool
get_and_check_add_size(dcontext_t *dcontext, fragment_t *f, uint *res_add_size,
uint *res_prev_mangle_size)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
uint add_size = f->size - fragment_prefix_size(f->flags) +
trace_exit_stub_size_diff(dcontext, f) +
(PAD_FRAGMENT_JMPS(md->trace_flags) ? extend_trace_pad_bytes(f) : 0);
* connect to this block if we were to add it
*/
uint prev_mangle_size = TRACE_CTI_MANGLE_SIZE_UPPER_BOUND;
uint total_size = md->emitted_size + add_size + prev_mangle_size;
bool ok = (total_size <= MAX_FRAGMENT_SIZE);
ASSERT(!TEST(FRAG_SELFMOD_SANDBOXED, f->flags));
ASSERT(!TEST(FRAG_IS_TRACE, f->flags));
LOG(THREAD, LOG_MONITOR, 4,
"checking trace size: currently %d, add estimate %d\n"
"\t(body: %d, stubs: %d, pad: %d, mangle est: %d)\n"
"\t=> %d vs %d, %d vs %d\n",
md->emitted_size, add_size + prev_mangle_size,
f->size - fragment_prefix_size(f->flags), trace_exit_stub_size_diff(dcontext, f),
(PAD_FRAGMENT_JMPS(md->trace_flags) ? extend_trace_pad_bytes(f) : 0),
prev_mangle_size, total_size, MAX_FRAGMENT_SIZE,
total_size * MAX_TRACE_FRACTION_OF_CACHE, DYNAMO_OPTION(cache_trace_max));
if (ok && DYNAMO_OPTION(cache_trace_max) > 0 &&
total_size * MAX_TRACE_FRACTION_OF_CACHE > DYNAMO_OPTION(cache_trace_max))
ok = false;
if (res_add_size != NULL)
*res_add_size = add_size;
if (res_prev_mangle_size != NULL)
*res_prev_mangle_size = prev_mangle_size;
return ok;
}
static inline uint
trace_flags_from_component_flags(uint flags)
{
return (flags &
(FRAG_HAS_SYSCALL |
FRAG_HAS_DIRECT_CTI IF_X86_64(
| FRAG_32_BIT IF_LINUX(| FRAG_HAS_RSEQ_ENDPOINT))));
}
static inline uint
trace_flags_from_trace_head_flags(uint head_flags)
{
uint trace_flags = 0;
if (!INTERNAL_OPTION(unsafe_ignore_eflags_prefix)) {
trace_flags |= (head_flags & FRAG_WRITES_EFLAGS_6);
trace_flags |= (head_flags & FRAG_WRITES_EFLAGS_OF);
}
trace_flags |= FRAG_IS_TRACE;
trace_flags |= trace_flags_from_component_flags(head_flags);
if (DYNAMO_OPTION(shared_traces)) {
trace_flags |= FRAG_SHARED;
}
return trace_flags;
}
* has the same tag as the one we're emitting as a trace.
*/
static fragment_t *
end_and_emit_trace(dcontext_t *dcontext, fragment_t *cur_f)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
fragment_t *trace_head_f = NULL;
app_pc tag = md->trace_tag;
app_pc cur_f_tag = cur_f->tag;
instrlist_t *trace = &md->trace;
fragment_t *trace_f;
trace_only_t *trace_tr;
bool replace_trace_head = false;
fragment_t wrapper;
uint i;
DEBUG_DECLARE(bool externally_mangled = false;)
* to a trace b/c traces have prefixes that basic blocks don't!
*/
DOSTATS({
if (LINKSTUB_INDIRECT(dcontext->last_exit->flags)) {
STATS_INC(num_traces_end_at_ibl);
if (EXIT_IS_CALL(dcontext->last_exit->flags)) {
STATS_INC(num_traces_end_at_ibl_ind_call);
} else if (EXIT_IS_JMP(dcontext->last_exit->flags)) {
if (IS_SHARED_SYSCALLS_LINKSTUB(dcontext->last_exit))
STATS_INC(num_traces_end_at_ibl_syscall);
else
STATS_INC(num_traces_end_at_ibl_ind_jump);
} else if (TEST(LINK_RETURN, dcontext->last_exit->flags)) {
STATS_INC(num_traces_end_at_ibl_return);
};
}
});
if (md->pass_to_client) {
* client, and we have to then re-mangle and re-connect.
*/
dr_emit_flags_t emitflags =
instrument_trace(dcontext, tag, &md->unmangled_ilist, false );
DODEBUG(externally_mangled = true;);
if (TEST(DR_EMIT_STORE_TRANSLATIONS, emitflags)) {
md->trace_flags |= FRAG_HAS_TRANSLATION_INFO;
}
if (!mangle_trace(dcontext, &md->unmangled_ilist, md)) {
trace_abort(dcontext);
STATS_INC(num_aborted_traces_client);
trace_f = NULL;
goto end_and_emit_trace_return;
}
instrlist_clear(dcontext, &md->trace);
md->trace = md->unmangled_ilist;
instrlist_init(&md->unmangled_ilist);
}
if (INTERNAL_OPTION(cbr_single_stub) &&
final_exit_shares_prev_stub(dcontext, trace, md->trace_flags)) {
* trace will also share -- here we find out and adjust
*/
instr_t *last = instrlist_last(trace);
app_pc target;
ASSERT(last != NULL && instr_is_exit_cti(last));
target = opnd_get_pc(instr_get_target(last));
md->emitted_size -= local_exit_stub_size(dcontext, target, md->trace_flags);
}
IF_AARCH64(md->emitted_size += fixup_indirect_trace_exit(dcontext, trace));
if (DYNAMO_OPTION(speculate_last_exit)
#ifdef HASHTABLE_STATISTICS
|| INTERNAL_OPTION(speculate_last_exit_stats) ||
INTERNAL_OPTION(stay_on_trace_stats)
#endif
) {
* only implemented for traces. If we have a sharable version
* of fixup_last_cti() to pass that information based on instr
* list information about last exit we can use in
* emit_fragment_common(). That way both bb's and traces may
* have speculation added.
*/
if (TEST(FRAG_MUST_END_TRACE, cur_f->flags)) {
* and in that case we haven't executed yet the last
* bb, so don't really know how to fix the last IBL
*/
ASSERT_CURIOSITY(dcontext->next_tag == cur_f->tag);
STATS_INC(num_traces_at_must_end_trace);
} else {
* is the current IBL target that we'll always speculate */
if (LINKSTUB_INDIRECT(dcontext->last_exit->flags)) {
LOG(THREAD, LOG_MONITOR, 2,
"Last trace IBL exit (trace " PFX ", next_tag " PFX ")\n", tag,
dcontext->next_tag);
ASSERT_CURIOSITY(dcontext->next_tag != NULL);
if (DYNAMO_OPTION(speculate_last_exit)) {
app_pc speculate_next_tag = dcontext->next_tag;
#ifdef SPECULATE_LAST_EXIT_STUDY
* all IBLs that never hit by comparing to a 0xbad tag */
speculate_next_tag = 0xbad;
#endif
md->emitted_size += append_trace_speculate_last_ibl(
dcontext, trace, speculate_next_tag, false);
} else {
#ifdef HASHTABLE_STATISTICS
ASSERT(INTERNAL_OPTION(stay_on_trace_stats) ||
INTERNAL_OPTION(speculate_last_exit_stats));
DOSTATS({
md->emitted_size += append_ib_trace_last_ibl_exit_stat(
dcontext, trace,
INTERNAL_OPTION(speculate_last_exit_stats)
? dcontext->next_tag
: NULL);
});
#endif
}
}
}
}
DOLOG(2, LOG_MONITOR, {
LOG(THREAD, LOG_MONITOR, 2, "Ending and emitting hot trace (tag " PFX ")\n", tag);
if (d_r_stats->loglevel >= 4) {
instrlist_disassemble(dcontext, md->trace_tag, trace, THREAD);
LOG(THREAD, LOG_MONITOR, 4, "\n");
}
LOG(THREAD, LOG_MONITOR, 2, "Trace blocks are:\n");
for (i = 0; i < md->num_blks; i++) {
LOG(THREAD, LOG_MONITOR, 2, "\tblock %3d == " PFX " (%d exit(s))\n", i,
md->blk_info[i].info.tag,
IF_RETURN_AFTER_CALL_ELSE(md->blk_info[i].info.num_exits, 0));
}
});
* must change recreate_app_state in arch/arch.c as well
*/
#ifdef INTERNAL
if (dynamo_options.optimize
# ifdef SIDELINE
&& !dynamo_options.sideline
# endif
) {
optimize_trace(dcontext, tag, trace);
DODEBUG(externally_mangled = true;);
}
#endif
#ifdef PROFILE_RDTSC
if (dynamo_options.profile_times) {
add_profile_call(dcontext);
}
#endif
#ifdef SIDELINE
if (dynamo_options.sideline) {
* ensure room in buffer and in cache
*/
add_sideline_prefix(dcontext, trace);
}
#endif
* for private traces:
* this way we use the head of FIFO for all our private copies, and
* then replace w/ the trace, avoiding any fragmentation from the copies.
* for shared traces: FIXME: case 5137: move temps to private bb cache?
*/
if (md->last_copy != NULL) {
if (cur_f == md->last_copy)
cur_f = NULL;
delete_private_copy(dcontext);
}
* We can't hold locks across cache executions, and we wouldn't want to have a
* massive trace building lock anyway, so we only grab a lock at the final emit
* moment and if there's a conflict the loser tosses his trace.
* We hold the lock across the trace head removal as well to avoid races there.
*/
if (TEST(FRAG_SHARED, md->trace_flags)) {
ASSERT(DYNAMO_OPTION(shared_traces));
d_r_mutex_lock(&trace_building_lock);
trace_f = fragment_lookup_trace(dcontext, tag);
if (trace_f != NULL) {
ASSERT(TEST(FRAG_IS_TRACE, trace_f->flags));
d_r_mutex_unlock(&trace_building_lock);
trace_abort(dcontext);
STATS_INC(num_aborted_traces_race);
#ifdef DEBUG
* practically all races (w/shared BBs anyway) much earlier.
* FIXME case 8769: we may need another way to prevent races w/
* -coarse_units!
*/
if (DYNAMO_OPTION(shared_bbs) && !DYNAMO_OPTION(coarse_units))
ASSERT_CURIOSITY(false);
#endif
goto end_and_emit_trace_return;
}
}
*
* For shared traces, if -no_remove_shared_trace_heads, we do not remove
* shared trace heads and only transfer their links
* over to the new trace (and if the trace is deleted we transfer the
* links back). We leave them alone otherwise, shadowed in both the DR
* lookup tables and ibl tables.
* FIXME: trace head left w/ no incoming -- will this break assumptions?
* What if someone who held ptr before trace emit, or does a different
* lookup, tries to mess w/ trace head's links?
*/
if (cur_f != NULL && cur_f->tag == tag) {
if (!TEST(FRAG_SHARED, cur_f->flags))
trace_head_f = cur_f;
* Set cur_f to NULL even if not deleted, since we want to
* execute the trace in preference to the trace head.
*/
cur_f = NULL;
}
if (trace_head_f == NULL)
trace_head_f = fragment_lookup_same_sharing(dcontext, tag, 0 );
* presuming that they have them for a reason and don't want this shared trace
*/
if (trace_head_f != NULL) {
LOG(THREAD, LOG_MONITOR, 4, "deleting private trace head fragment\n");
* call monitor_remove_fragment() to avoid aborting our trace
*/
if (trace_head_f == dcontext->last_fragment)
last_exit_deleted(dcontext);
* shadow it. If the trace is shared, we have to delete it. We'll re-create
* the head as a shared bb if we ever do build a custom trace through it.
*/
if (!TEST(FRAG_SHARED, md->trace_flags)) {
replace_trace_head = true;
CLIENT_ASSERT(!DYNAMO_OPTION(shared_bbs),
"invalid private trace head and "
"private traces but -shared_bbs for custom traces");
} else {
fragment_delete(dcontext, trace_head_f,
FRAGDEL_NO_OUTPUT | FRAGDEL_NO_MONITOR);
}
if (!replace_trace_head) {
trace_head_f = NULL;
STATS_INC(num_fragments_deleted_trace_heads);
}
}
if (DYNAMO_OPTION(shared_bbs)) {
trace_head_f = fragment_lookup_fine_and_coarse_sharing(dcontext, tag, &wrapper,
NULL, FRAG_SHARED);
if (!TEST(FRAG_SHARED, md->trace_flags)) {
} else if (trace_head_f != NULL) {
* with trace head being re-created before the trace is visible
*/
replace_trace_head = true;
if (!TEST(FRAG_IS_TRACE_HEAD, trace_head_f->flags)) {
ASSERT(TEST(FRAG_COARSE_GRAIN, trace_head_f->flags));
trace_head_f->flags |= FRAG_IS_TRACE_HEAD;
}
}
}
* cache as our trace (esp. w/ a MUST_END_TRACE trace head, since then the
* last_fragment can be another trace) from clobbering our trace!
* FIXME: would be cleaner to remove the need to abort the trace if
* last_fragment is deleted, but tricky to do that (see
* monitor_remove_fragment). Could also use a special monitor_data_t field
* saying "ignore last_exit, I'm emitting now."
*/
if (!LINKSTUB_FAKE(dcontext->last_exit))
last_exit_deleted(dcontext);
ASSERT(md->last_fragment == NULL);
ASSERT(md->last_copy == NULL);
ASSERT(md->trace_tag == tag);
if (replace_trace_head) {
trace_f = emit_fragment_as_replacement(dcontext, tag, trace, md->trace_flags,
md->trace_vmlist, trace_head_f);
} else {
trace_f = emit_fragment(dcontext, tag, trace, md->trace_flags, md->trace_vmlist,
true );
}
ASSERT(trace_f != NULL);
* if externally mangled, all bets are off for now --
* FIXME: would be nice to gracefully handle opt or client
* making the trace too big, and pass back an error msg?
* Perhaps have lower size bounds when optimization or client
* interface are on.
*/
LOG(THREAD, LOG_MONITOR, 3, "Trace estimated size %d vs actual size %d\n",
md->emitted_size, trace_f->size);
ASSERT(trace_f->size <= md->emitted_size || externally_mangled);
* by how much though FIXME */
ASSERT_CURIOSITY(trace_f->size == md->emitted_size || externally_mangled ||
PAD_FRAGMENT_JMPS(trace_f->flags));
trace_tr = TRACE_FIELDS(trace_f);
trace_tr->num_bbs = md->num_blks;
trace_tr->bbs = (trace_bb_info_t *)nonpersistent_heap_alloc(
FRAGMENT_ALLOC_DC(dcontext, trace_f->flags),
md->num_blks * sizeof(trace_bb_info_t) HEAPACCT(ACCT_TRACE));
for (i = 0; i < md->num_blks; i++)
trace_tr->bbs[i] = md->blk_info[i].info;
if (TEST(FRAG_SHARED, md->trace_flags))
d_r_mutex_unlock(&trace_building_lock);
RSTATS_INC(num_traces);
DOSTATS(
{ IF_X86_64(if (FRAG_IS_32(trace_f->flags)) { STATS_INC(num_32bit_traces); }) });
STATS_ADD(num_bbs_in_all_traces, md->num_blks);
STATS_TRACK_MAX(max_bbs_in_a_trace, md->num_blks);
DOLOG(2, LOG_MONITOR, {
LOG(THREAD, LOG_MONITOR, 1, "Generated trace fragment #%d for tag " PFX "\n",
GLOBAL_STAT(num_traces), tag);
disassemble_fragment(dcontext, trace_f, d_r_stats->loglevel < 3);
});
#ifdef INTERNAL
DODEBUG({
if (INTERNAL_OPTION(stress_recreate_pc)) {
stress_test_recreate(dcontext, trace_f, trace);
}
});
#endif
* but we must clear these two before enter_nolinking so that a flusher
* doesn't access them in an inconsistent state (trace_vmlist is invalid
* once also pointers are transferred to real fragment)
*/
md->trace_vmlist = NULL;
md->trace_tag = NULL;
* nolinking, meaning we need to hold no locks here, and that when
* we get back our local fragment_t pointers may be invalid.
*/
if (trace_head_f != NULL && DYNAMO_OPTION(shared_bbs) &&
TEST(FRAG_SHARED, md->trace_flags) &&
* If we were to remove it we would need a solution to finding it for
* pc translation, which currently walks the htable.
*/
!TEST(FRAG_COARSE_GRAIN, trace_head_f->flags) &&
!dr_end_trace_hook_exists() && INTERNAL_OPTION(remove_shared_trace_heads)) {
fragment_remove_shared_no_flush(dcontext, trace_head_f);
trace_head_f = NULL;
}
if (DYNAMO_OPTION(remove_trace_components)) {
fragment_t *f;
for (i = 1 ; i < md->num_blks; i++) {
f = fragment_lookup_bb(dcontext, md->blk_info[i].info.tag);
if (f != NULL) {
if (TEST(FRAG_SHARED, f->flags) && !TEST(FRAG_COARSE_GRAIN, f->flags)) {
fragment_remove_shared_no_flush(dcontext, f);
trace_head_f = NULL;
} else
fragment_delete(dcontext, f, FRAGDEL_NO_OUTPUT | FRAGDEL_NO_MONITOR);
STATS_INC(trace_components_deleted);
}
}
}
instrlist_clear(dcontext, trace);
md->trace_tag = tag;
reset_trace_state(dcontext, true );
#ifdef DEBUG
* shouldn't be set on the trace head fragment. If we're not using shared
* BBs or are not building shared traces, the flag shouldn't be set then
* either. Basically, it should never be set at this point, after the call
* to reset_trace_state() just above.
*/
if (trace_head_f != NULL)
ASSERT(!TEST(FRAG_TRACE_BUILDING, trace_head_f->flags));
#endif
end_and_emit_trace_return:
if (cur_f == NULL && cur_f_tag == tag)
return trace_f;
else {
* best way to find out is to re-look-up the next fragment (this only
* happens when emitting trace, so rare enough)
*/
cur_f = fragment_lookup(dcontext, cur_f_tag);
return cur_f;
}
}
* internal_extend_trace() rather than the next time into monitor_cache_enter()
* if fragment results in a system call (sysenter) or callback (int 2b), i.e.,
* is marked FRAG_MUST_END_TRACE.
*/
static fragment_t *
internal_extend_trace(dcontext_t *dcontext, fragment_t *f, linkstub_t *prev_l,
uint add_size)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
bool have_locks = false;
DEBUG_DECLARE(uint pre_emitted_size = md->emitted_size;)
extend_unmangled_ilist(dcontext, f);
if (is_ibl_sourceless_linkstub((const linkstub_t *)prev_l)) {
ASSERT(!DYNAMO_OPTION(indirect_stubs));
prev_l = NULL;
}
ASSERT(prev_l == NULL || !LINKSTUB_FAKE(prev_l) ||
prev_l == get_deleted_linkstub(dcontext));
if (TEST(FRAG_SHARED, f->flags)) {
* fragment_remove_shared_no_flush()-ed underneath us, eliminating its
* also fields needed for vm_area_add_to_list() (plus w/ the also field
* re-used for case 3559 we have crash potential).
*/
have_locks = true;
SHARED_FLAGS_RECURSIVE_LOCK(f->flags, acquire, change_linking_lock);
acquire_vm_areas_lock(dcontext, f->flags);
}
if (TEST(FRAG_WAS_DELETED, f->flags)) {
* fragment_t.also is now invalid!
*/
STATS_INC(num_trace_next_bb_deleted);
ASSERT(have_locks);
if (have_locks) {
release_vm_areas_lock(dcontext, f->flags);
SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
}
return end_and_emit_trace(dcontext, f);
}
* have that passed in, though without the estimate for the mangling
* of the previous block (thus including only f->size and the exit stub
* size changes), which we calculate in extend_trace.
* Existing custom stub code should already be in f->size.
* FIXME: if we ever have decode_fragment() convert, say, dcontext
* save/restore to tls, then we'll have to add in its size increases
* as well.
*/
md->emitted_size += add_size;
md->trace_flags |= trace_flags_from_component_flags(f->flags);
md->emitted_size += extend_trace(dcontext, f, prev_l);
LOG(THREAD, LOG_MONITOR, 3, "extending added %d to size of trace => %d total\n",
md->emitted_size - pre_emitted_size, md->emitted_size);
vm_area_add_to_list(dcontext, md->trace_tag, &(md->trace_vmlist), md->trace_flags, f,
have_locks);
if (have_locks) {
* create_private_copy (it calls emit()) but we're at a stable state
* now.
*/
release_vm_areas_lock(dcontext, f->flags);
SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
}
DOLOG(3, LOG_MONITOR, {
LOG(THREAD, LOG_MONITOR, 4, "After extending, trace looks like this:\n");
instrlist_disassemble(dcontext, md->trace_tag, &md->trace, THREAD);
});
* trace aborts due to syscalls and callbacks. See case 3541.
*/
if (TEST(FRAG_MUST_END_TRACE, f->flags)) {
* prevent its deletion during emitting from clobbering the trace in the case
* that last_fragment==f (requires that f targets itself, and f is
* private like traces -- not possible w/ today's syscall-only MUST_END_TRACE
* fragments but could happen in the future) -- except that that's a general
* problem handled by clearing last_exit in end_and_emit_trace, so we do
* nothing here.
*/
return end_and_emit_trace(dcontext, f);
}
ASSERT(!TEST(FRAG_SHARED, f->flags));
if (TEST(FRAG_TEMP_PRIVATE, f->flags)) {
* thread private fragment.
*/
ASSERT(md->last_fragment == f);
ASSERT(md->last_copy != NULL);
ASSERT(md->last_copy->tag == f->tag);
ASSERT(md->last_fragment == md->last_copy);
} else {
* to restore. can't rely on last_exit for restoring since could end up
* not coming out of cache from last_fragment (e.g., if hit sigreturn)
*/
md->last_fragment = f;
}
SHARED_FLAGS_RECURSIVE_LOCK(f->flags, acquire, change_linking_lock);
md->last_fragment_flags = f->flags;
if ((f->flags & FRAG_CANNOT_DELETE) == 0) {
* dcontext->last_exit for extend_trace
*/
f->flags |= FRAG_CANNOT_DELETE;
LOG(THREAD, LOG_MONITOR, 4, "monitor marked F%d (" PFX ") as un-deletable\n",
f->id, f->tag);
}
if ((f->flags & FRAG_LINKED_OUTGOING) != 0) {
unlink_fragment_outgoing(dcontext, f);
LOG(THREAD, LOG_MONITOR | LOG_LINKS, 4, "monitor unlinked F%d (" PFX ")\n", f->id,
f->tag);
}
SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
return f;
}
* it's a field used only by us.
*/
static void
internal_restore_last(dcontext_t *dcontext)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
if (md->last_fragment == NULL)
return;
* sometimes we come in here from trace_abort and we've already restored
* the last exit, so check before linking.
*/
SHARED_FLAGS_RECURSIVE_LOCK(md->last_fragment->flags, acquire, change_linking_lock);
if ((md->last_fragment_flags & FRAG_LINKED_OUTGOING) != 0 &&
(md->last_fragment->flags & FRAG_LINKED_OUTGOING) == 0) {
LOG(THREAD, LOG_MONITOR, 4, "internal monitor: relinking last fragment F%d\n",
md->last_fragment->id);
link_fragment_outgoing(dcontext, md->last_fragment, false);
}
if ((md->last_fragment_flags & FRAG_CANNOT_DELETE) == 0 &&
(md->last_fragment->flags & FRAG_CANNOT_DELETE) != 0) {
LOG(THREAD, LOG_MONITOR, 4,
"internal monitor: re-marking last fragment F%d as deletable\n",
md->last_fragment->id);
md->last_fragment->flags &= ~FRAG_CANNOT_DELETE;
}
ASSERT((md->last_fragment_flags & (FRAG_CANNOT_DELETE | FRAG_LINKED_OUTGOING)) ==
(md->last_fragment->flags & (FRAG_CANNOT_DELETE | FRAG_LINKED_OUTGOING)));
SHARED_FLAGS_RECURSIVE_LOCK(md->last_fragment->flags, release, change_linking_lock);
* deletion of trace head will cause use to abort single-bb trace
* (see monitor_remove_fragment)
*
* Do NOT reset last_fragment_flags as that field is needed prior to the
* cache entry and is referenced in monitor_cache_enter().
*/
if (!TEST(FRAG_TEMP_PRIVATE, md->last_fragment->flags))
md->last_fragment = NULL;
}
void
monitor_cache_exit(dcontext_t *dcontext)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
ASSERT(dcontext->whereami == DR_WHERE_DISPATCH);
dcontext->whereami = DR_WHERE_MONITOR;
if (md->trace_tag != NULL && md->last_fragment != NULL) {
SELF_PROTECT_LOCAL(dcontext, WRITABLE);
internal_restore_last(dcontext);
SELF_PROTECT_LOCAL(dcontext, READONLY);
} else if (md->trace_tag == NULL) {
* non-ignorable syscall that used the SYSENTER instruction, which
* we've seen on XP and 2003. The 'ret' after the SYSENTER executes
* natively, and this piece of control flow isn't captured during
* linking so link-time trace head marking doesn't work. (The exit
* stub is marked as a direct exit.) The exit stub is reset during
* syscall handling so indirect-exit trace head marking isn't
* possible either, so we have to use a dedicated var to capture
* this case.
*
* We need to set trace_sysenter_exit to true or false to prevent a
* stale value from reaching a later read of the flag.
*
* FIXME Rework this to store the last (pre-syscall) exit's fragment flags & tag
* in a dcontext-private place such as non-shared monitor data.
* Such a general mechanism will permit us to capture all
* trace head marking within should_be_trace_head().
*/
dcontext->trace_sysenter_exit =
(TEST(FRAG_IS_TRACE, dcontext->last_fragment->flags) &&
TEST(LINK_NI_SYSCALL, dcontext->last_exit->flags));
}
dcontext->whereami = DR_WHERE_DISPATCH;
}
static void
check_fine_to_coarse_trace_head(dcontext_t *dcontext, fragment_t *f)
{
* no way to indicate that (there is no entrance stub for the fine
* fragments, as once the coarse unit is frozen we can't use its
* entrance stub). So we assume that an exit is due to trace headness
* discovered at link time iff it would now be considered a trace head.
* FIXME: any cleaner way?
*/
if (TEST(FRAG_COARSE_GRAIN, f->flags) && !TEST(FRAG_IS_TRACE_HEAD, f->flags) &&
* fragments. Oh well.
*/
!TESTANY(FRAG_COARSE_GRAIN | FRAG_FAKE, dcontext->last_fragment->flags)) {
acquire_recursive_lock(&change_linking_lock);
if (check_for_trace_head(dcontext, dcontext->last_fragment, dcontext->last_exit,
f, true , false )) {
STATS_INC(num_exits_fine2th_coarse);
} else {
* bb and may target a coarse bb
*/
STATS_INC(num_exits_fine2non_th_coarse);
}
release_recursive_lock(&change_linking_lock);
}
}
* regions, and it controls the building and installation of trace
* fragments.
*/
fragment_t *
monitor_cache_enter(dcontext_t *dcontext, fragment_t *f)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
bool start_trace = false;
bool end_trace = false;
dr_custom_trace_action_t client = CUSTOM_TRACE_DR_DECIDES;
trace_head_counter_t *ctr;
uint add_size = 0, prev_mangle_size = 0;
if (DYNAMO_OPTION(disable_traces) || f == NULL) {
ASSERT(md->trace_tag == NULL);
return f;
}
ASSERT(dcontext->whereami == DR_WHERE_DISPATCH);
dcontext->whereami = DR_WHERE_MONITOR;
* (xref bug 8637 on not terminating traces b/c we marked as head too late)
*/
check_fine_to_coarse_trace_head(dcontext, f);
if (md->trace_tag != NULL) {
KSTART(trace_building);
SELF_PROTECT_LOCAL(dcontext, WRITABLE);
ASSERT(md->last_fragment == NULL ||
TEST(FRAG_TEMP_PRIVATE, md->last_fragment->flags));
end_trace = (end_trace || TEST(FRAG_IS_TRACE, f->flags) ||
TEST(FRAG_IS_TRACE_HEAD, f->flags));
if (dr_end_trace_hook_exists()) {
client = instrument_end_trace(dcontext, md->trace_tag, f->tag);
* CUSTOM_TRACE_DR_DECIDES = use standard termination criteria
* CUSTOM_TRACE_END_NOW = end trace
* CUSTOM_TRACE_CONTINUE = do not end trace
*/
if (client == CUSTOM_TRACE_END_NOW) {
DOSTATS({
if (!end_trace) {
LOG(THREAD, LOG_MONITOR, 3,
"Client ending 0x%08x trace early @0x%08x\n", md->trace_tag,
f->tag);
STATS_INC(custom_traces_stop_early);
}
});
end_trace = true;
} else if (client == CUSTOM_TRACE_CONTINUE) {
DOSTATS({
if (end_trace) {
LOG(THREAD, LOG_MONITOR, 3,
"Client not ending 0x%08x trace @ normal stop @0x%08x\n",
md->trace_tag, f->tag);
STATS_INC(custom_traces_stop_late);
}
});
end_trace = false;
}
LOG(THREAD, LOG_MONITOR, 4, "Client instrument_end_trace returned %d\n",
client);
}
end_trace = end_trace || TEST(FRAG_CANNOT_BE_TRACE, f->flags);
#if defined(X86) && defined(X64)
if (TEST(FRAG_32_BIT, f->flags) != TEST(FRAG_32_BIT, md->trace_flags))
end_trace = true;
#endif
if (!end_trace) {
if (TEST(FRAG_IS_TRACE, f->flags)) {
* hook, right?). We do not link the new, shadowed bb.
*/
fragment_t *head = NULL;
if (USE_BB_BUILDING_LOCK())
d_r_mutex_lock(&bb_building_lock);
if (DYNAMO_OPTION(coarse_units)) {
* a custom lookup
*/
head = fragment_coarse_lookup_wrapper(dcontext, f->tag, &md->wrapper);
}
if (head == NULL)
head = fragment_lookup_bb(dcontext, f->tag);
if (head == NULL) {
LOG(THREAD, LOG_MONITOR, 3,
"Client custom trace 0x%08x requiring shadow bb 0x%08x\n",
md->trace_tag, f->tag);
SELF_PROTECT_LOCAL(dcontext, WRITABLE);
* and asserts when adding to fragment htable and unlinking
* on delete.
*/
head = build_basic_block_fragment(
dcontext, f->tag, FRAG_IS_TRACE_HEAD, false ,
true , true , NULL);
SELF_PROTECT_LOCAL(dcontext, READONLY);
STATS_INC(custom_traces_bbs_built);
ASSERT(head != NULL);
* We shouldn't end up w/ a bb of different sharing than the
* trace: custom traces rule out private traces and shared bbs,
* and if circumstances changed since the original trace head bb
* was made then the trace should have been flushed.
*/
ASSERT((head->flags & FRAG_SHARED) == (f->flags & FRAG_SHARED));
if (TEST(FRAG_COARSE_GRAIN, head->flags)) {
* FIXME: share this code sequence w/ d_r_dispatch().
*/
ASSERT(USE_BB_BUILDING_LOCK());
fragment_coarse_wrapper(&md->wrapper, f->tag,
FCACHE_ENTRY_PC(head));
md->wrapper.flags |= FRAG_IS_TRACE_HEAD;
head = &md->wrapper;
}
}
if (USE_BB_BUILDING_LOCK())
d_r_mutex_unlock(&bb_building_lock);
f = head;
}
if (TEST(FRAG_COARSE_GRAIN, f->flags) || TEST(FRAG_SHARED, f->flags) ||
md->pass_to_client) {
* ahead and make a private copy here.
* For shared fragments, we make a private copy of f to avoid
* synch issues with other threads modifying its linkage before
* we get back here. We do it up front now (i#940) to avoid
* determinism issues that arise when check_thread_vm_area()
* changes its mind over time.
*/
create_private_copy(dcontext, f);
if (md->trace_tag == NULL) {
* someone (see comments in create_private_copy) --
* when emitting our private bb we can kill the
* last_fragment): just exit now
*/
LOG(THREAD, LOG_MONITOR, 4,
"Private copy ended up aborting trace!\n");
STATS_INC(num_trace_private_copy_abort);
* undid the clearing of last_fragment by assigning it
* to last_copy, must re-clear!
*/
md->last_fragment = NULL;
KSTOP(trace_building);
return f;
}
f = md->last_fragment;
}
if (!end_trace &&
!get_and_check_add_size(dcontext, f, &add_size, &prev_mangle_size)) {
STATS_INC(num_max_trace_size_enforced);
end_trace = true;
}
}
if (DYNAMO_OPTION(max_trace_bbs) > 0 &&
md->num_blks >= DYNAMO_OPTION(max_trace_bbs) && !end_trace) {
end_trace = true;
STATS_INC(num_max_trace_bbs_enforced);
}
end_trace =
(end_trace ||
!make_room_in_trace_buffer(dcontext, add_size + prev_mangle_size, f));
if (end_trace && client == CUSTOM_TRACE_CONTINUE) {
LOG(THREAD, LOG_MONITOR, 2,
PRODUCT_NAME
" ignoring Client's decision to "
"continue trace (cannot trace through next fragment), ending trace "
"now\n");
}
if (end_trace) {
LOG(THREAD, LOG_MONITOR, 3,
"NOT extending hot trace (tag " PFX ") with F%d (" PFX ")\n",
md->trace_tag, f->id, f->tag);
f = end_and_emit_trace(dcontext, f);
LOG(THREAD, LOG_MONITOR, 3, "Returning to search mode f=" PFX "\n", f);
} else {
LOG(THREAD, LOG_MONITOR, 3,
"Extending hot trace (tag " PFX ") with F%d (" PFX ")\n", md->trace_tag,
f->id, f->tag);
f = internal_extend_trace(dcontext, f, dcontext->last_exit, add_size);
}
dcontext->whereami = DR_WHERE_DISPATCH;
SELF_PROTECT_LOCAL(dcontext, READONLY);
KSTOP(trace_building);
return f;
}
if (TEST(FRAG_IS_TRACE, f->flags)) {
dcontext->whereami = DR_WHERE_DISPATCH;
return f;
}
if (!TEST(FRAG_IS_TRACE_HEAD, f->flags)) {
bool trace_head;
* - indirect exits
* - an exit from a trace that ends just before a SYSENTER.
* - private secondary trace heads targeted by shared traces
*
* FIXME Rework this to use the last exit's fragment flags & tag that were
* stored in a dcontext-private place such as non-shared monitor data.
*/
if (LINKSTUB_INDIRECT(dcontext->last_exit->flags) ||
dcontext->trace_sysenter_exit ||
(TESTALL(FRAG_SHARED | FRAG_IS_TRACE, dcontext->last_fragment->flags) &&
!TESTANY(FRAG_SHARED | FRAG_IS_TRACE, f->flags))) {
bool need_lock = NEED_SHARED_LOCK(dcontext->last_fragment->flags);
if (need_lock)
acquire_recursive_lock(&change_linking_lock);
* path thru check_for_trace_head() accounts for that.
*/
trace_head = check_for_trace_head(dcontext, dcontext->last_fragment,
dcontext->last_exit, f, need_lock,
dcontext->trace_sysenter_exit);
if (need_lock)
release_recursive_lock(&change_linking_lock);
* entire fcache
*/
SELF_PROTECT_CACHE(dcontext, NULL, READONLY);
} else {
trace_head = false;
}
if (!trace_head) {
dcontext->whereami = DR_WHERE_DISPATCH;
return f;
}
}
ctr = thcounter_lookup(dcontext, f->tag);
if (ctr == NULL)
ctr = thcounter_add(dcontext, f->tag);
ASSERT(ctr != NULL);
if (ctr->counter == TH_COUNTER_CREATED_TRACE_VALUE()) {
* deletion. However, when a trace is deleted we clear the counter, to
* prevent the new bb from immediately being considered hot, to help
* with phased execution (trace may no longer be hot). To avoid having
* walk every thread for every trace deleted we use a lazy strategy,
* recognizing a counter that has already reached the threshold with a
* sentinel value.
*/
ctr->counter = INTERNAL_OPTION(trace_counter_on_delete);
STATS_INC(th_counter_reset);
}
ctr->counter++;
if (ctr->counter >= INTERNAL_OPTION(trace_threshold)) {
* can delete it (w/ exceptions, deletion status changes). */
if (!TEST(FRAG_CANNOT_DELETE, f->flags)) {
if (!DYNAMO_OPTION(shared_traces))
start_trace = true;
* we need a presence table to mark that a tag is being used for trace
* building. Generic hashtables can help with this (case 6206).
*/
else if (!DYNAMO_OPTION(shared_bbs) || !TEST(FRAG_SHARED, f->flags))
start_trace = true;
else {
ASSERT(TEST(FRAG_SHARED, f->flags));
acquire_recursive_lock(&change_linking_lock);
if (TEST(FRAG_TRACE_BUILDING, f->flags)) {
LOG(THREAD, LOG_MONITOR, 3,
"Not going to start trace with already-in-trace-progress F%d "
"(tag " PFX ")\n",
f->id, f->tag);
STATS_INC(num_trace_building_race);
} else {
LOG(THREAD, LOG_MONITOR, 3,
"Going to start trace with F%d (tag " PFX ")\n", f->id, f->tag);
f->flags |= FRAG_TRACE_BUILDING;
start_trace = true;
}
release_recursive_lock(&change_linking_lock);
}
}
}
if (start_trace) {
* to be consistent across whole trace. If client later unregisters bb
* hook then it will miss our call on constituent bbs: that's its problem.
* We document that trace and bb hooks should not be unregistered.
*/
md->pass_to_client = mangle_trace_at_end();
ASSERT(instrlist_first(&md->unmangled_ilist) == NULL);
}
if (start_trace &&
(TEST(FRAG_COARSE_GRAIN, f->flags) || TEST(FRAG_SHARED, f->flags) ||
md->pass_to_client)) {
ASSERT(TEST(FRAG_IS_TRACE_HEAD, f->flags));
* ahead and make a private copy here.
* For shared fragments, we make a private copy of f to avoid
* synch issues with other threads modifying its linkage before
* we get back here. We do it up front now (i#940) to avoid
* determinism issues that arise when check_thread_vm_area()
* changes its mind over time.
*/
create_private_copy(dcontext, f);
f = md->last_fragment;
}
if (!start_trace && ctr->counter >= INTERNAL_OPTION(trace_threshold)) {
* counter will be == trace_threshold if this thread is later
* able to start building a trace w/this tag and ensures
* that our one-up sentinel works for lazy clearing.
*/
LOG(THREAD, LOG_MONITOR, 3, "Backing up F%d counter from %d\n", f->id,
ctr->counter);
ctr->counter--;
ASSERT(ctr->counter < INTERNAL_OPTION(trace_threshold));
}
if (start_trace) {
KSTART(trace_building);
ASSERT(ctr->counter == INTERNAL_OPTION(trace_threshold));
ctr->counter = TH_COUNTER_CREATED_TRACE_VALUE();
selection mode, and initialize the instrlist_t for the new
trace fragment with this block fragment. Leave the
trace head entry locked so no one else tries to build
a trace from it. Assume that a trace would never
contain just one block, and thus we don't have to check
for end of trace condition here. */
SELF_PROTECT_LOCAL(dcontext, WRITABLE);
#ifdef TRACE_HEAD_CACHE_INCR
* in the future since we can only encounter the trace head in our
* no-link trace-building mode, then we will delete it
*/
#endif
md->trace_tag = f->tag;
md->trace_flags = trace_flags_from_trace_head_flags(f->flags);
md->emitted_size = fragment_prefix_size(md->trace_flags);
#ifdef PROFILE_RDTSC
if (dynamo_options.profile_times)
md->emitted_size += profile_call_size();
#endif
LOG(THREAD, LOG_MONITOR, 2, "Found hot trace head F%d (tag " PFX ")\n", f->id,
f->tag);
LOG(THREAD, LOG_MONITOR, 3, "Entering trace selection mode\n");
ASSERT(!TEST(FRAG_IS_TRACE, f->flags));
if (!get_and_check_add_size(dcontext, f, &add_size, &prev_mangle_size) ||
!make_room_in_trace_buffer(
dcontext, md->emitted_size + add_size + prev_mangle_size, f)) {
LOG(THREAD, LOG_MONITOR, 1, "bb %d (" PFX ") too big (%d) %s\n", f->id,
f->tag, f->size,
get_and_check_add_size(dcontext, f, NULL, NULL)
? "trace buffer"
: "trace body limit / trace cache size");
SHARED_FLAGS_RECURSIVE_LOCK(f->flags, acquire, change_linking_lock);
f->flags &= ~FRAG_IS_TRACE_HEAD;
f->flags |= FRAG_CANNOT_BE_TRACE;
STATS_INC(num_huge_fragments);
link_fragment_incoming(dcontext, f, false );
* may manipulate frag flags */
reset_trace_state(dcontext, false );
SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
* export the size expansion factors considered?
*/
dcontext->whereami = DR_WHERE_DISPATCH;
SELF_PROTECT_CACHE(dcontext, NULL, READONLY);
SELF_PROTECT_LOCAL(dcontext, READONLY);
KSTOP(trace_building);
return f;
}
f = internal_extend_trace(dcontext, f, NULL, add_size);
SELF_PROTECT_LOCAL(dcontext, READONLY);
KSTOP(trace_building);
} else {
KSWITCH(monitor_enter_thci);
}
dcontext->whereami = DR_WHERE_DISPATCH;
return f;
}
* to call from any linking state. Restores linking to previous state at exit.
* If calling on another thread, caller should be synchronized with that thread
* (either via flushing synch or thread_synch methods) FIXME : verify all users
* on other threads are properly synchronized
*/
void
trace_abort(dcontext_t *dcontext)
{
monitor_data_t *md = (monitor_data_t *)dcontext->monitor_field;
instrlist_t *trace;
bool prevlinking = true;
if (md->trace_tag == NULL && md->last_copy == NULL)
return;
* non-could-be-linking locations, so we synch w/ flusher here.
* additionally we are changing trace state that the flusher
* reads, and we could have a race condition, so we consider
* that to be a linking change as well. If we are the flusher
* then the synch is unnecessary and could even cause a livelock.
*/
if (!is_self_flushing()) {
if (!is_couldbelinking(dcontext)) {
prevlinking = false;
enter_couldbelinking(dcontext, NULL, false );
}
}
* cannot use last_exit, must use our own last_fragment just for this
* purpose, b/c may not exit cache from last_fragment
* (e.g., if hit sigreturn!)
*/
if (md->last_fragment != NULL) {
internal_restore_last(dcontext);
}
* in that fragment. For example, a client could have a clean call that
* flushes. We'll delete the last_copy when we start the next trace or at
* thread exit instead.
*/
trace = &md->trace;
instrlist_clear(dcontext, trace);
if (md->trace_vmlist != NULL) {
vm_area_destroy_list(dcontext, md->trace_vmlist);
md->trace_vmlist = NULL;
}
STATS_INC(num_aborted_traces);
STATS_ADD(num_bbs_in_all_aborted_traces, md->num_blks);
reset_trace_state(dcontext, true );
if (!prevlinking)
enter_nolinking(dcontext, NULL, false );
}
#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
app_pc
get_trace_exit_component_tag(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
{
linkstub_t *stub;
uint exitnum = 0;
uint i, num;
app_pc tag = f->tag;
bool found = false;
trace_only_t *t = TRACE_FIELDS(f);
ASSERT(TEST(FRAG_IS_TRACE, f->flags));
ASSERT(linkstub_fragment(dcontext, l) == f);
for (stub = FRAGMENT_EXIT_STUBS(f); stub != NULL; stub = LINKSTUB_NEXT_EXIT(stub)) {
if (stub == l) {
found = true;
break;
}
exitnum++;
}
ASSERT(found);
if (!found) {
LOG(THREAD, LOG_MONITOR, 2,
"get_trace_exit_component_tag F%d(" PFX "): can't find exit!\n", f->id,
f->tag);
return f->tag;
}
ASSERT(exitnum < t->num_bbs);
* but we will during trace building. Rather than recreate each bb and figure
* out how many exits it contributed, we store that information.
*/
found = false;
for (i = 0, num = 0; i < t->num_bbs; i++) {
if (exitnum < num + t->bbs[i].num_exits) {
found = true;
tag = t->bbs[i].tag;
break;
}
num += t->bbs[i].num_exits;
}
ASSERT(found);
LOG(THREAD, LOG_MONITOR, 4,
"get_trace_exit_component_tag F%d(" PFX ") => bb #%d (exit #%d): " PFX "\n",
f->id, f->tag, i, exitnum, tag);
return tag;
}
#endif