Thanks to visit codestin.com
Credit goes to github.com

Skip to content

[profiler] Fix thread list insertion failures. #6115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kumpera merged 4 commits into from
Nov 29, 2017
Merged

[profiler] Fix thread list insertion failures. #6115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
832e161
[profiler] Add thread_stopping and thread_exited profiler events.
Nov 28, 2017
cd46e14
[profiler] Use thread_exited profiler event for recording thread end …
Nov 28, 2017
22d22ff
[profiler] Remove tools_thread checks when raising profiler thread ev…
Nov 28, 2017
c36f5af
[profiler] Add an assertion to the log profiler to catch the thread r…
Nov 28, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions mono/metadata/profiler-events.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,9 @@ MONO_PROFILER_EVENT_1(monitor_failed, MonitorFailed, MonoObject *, object)
MONO_PROFILER_EVENT_1(monitor_acquired, MonitorAcquired, MonoObject *, object)

MONO_PROFILER_EVENT_1(thread_started, ThreadStarted, uintptr_t, tid)
MONO_PROFILER_EVENT_1(thread_stopping, ThreadStopping, uintptr_t, tid)
MONO_PROFILER_EVENT_1(thread_stopped, ThreadStopped, uintptr_t, tid)
MONO_PROFILER_EVENT_1(thread_exited, ThreadExited, uintptr_t, tid)
MONO_PROFILER_EVENT_2(thread_name, ThreadName, uintptr_t, tid, const char *, name)

MONO_PROFILER_EVENT_2(sample_hit, SampleHit, const mono_byte *, ip, const void *, context)
Expand Down
15 changes: 6 additions & 9 deletions mono/metadata/threads.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
#include <mono/utils/mono-time.h>
#include <mono/utils/mono-threads.h>
#include <mono/utils/mono-threads-coop.h>
#include <mono/utils/hazard-pointer.h>
#include <mono/utils/mono-tls.h>
#include <mono/utils/atomic.h>
#include <mono/utils/mono-memory-model.h>
Expand Down Expand Up @@ -749,6 +748,8 @@ mono_thread_detach_internal (MonoInternalThread *thread)

THREAD_DEBUG (g_message ("%s: mono_thread_detach for %p (%"G_GSIZE_FORMAT")", __func__, thread, (gsize)thread->tid));

MONO_PROFILER_RAISE (thread_stopping, (thread->tid));

#ifndef HOST_WIN32
mono_w32mutex_abandon ();
#endif
Expand Down Expand Up @@ -845,11 +846,7 @@ mono_thread_detach_internal (MonoInternalThread *thread)

mono_release_type_locks (thread);

/* Can happen when we attach the profiler helper thread in order to heapshot. */
if (!mono_thread_info_lookup (MONO_UINT_TO_NATIVE_THREAD_ID (thread->tid))->tools_thread)
Copy link
Contributor

@kumpera kumpera Nov 28, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting change, cuz the old code was incorrect.
MONO_PROFILER_RAISE uses hazard pointers in the same way as mono_thread_info_lookup, so whatever we need to be protected would be overwritten.

Copy link
Contributor Author

@alexrp alexrp Nov 28, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, the callbacks invoked by MONO_PROFILER_RAISE may or may not use hazard pointers. The mono_hazard_pointer_clear below was there to clear HP1 (which would have been set by mono_thread_info_lookup) just in case the profiler callbacks wouldn't have overwritten it with something else. Was that approach actually problematic?

MONO_PROFILER_RAISE (thread_stopped, (thread->tid));

mono_hazard_pointer_clear (mono_hazard_pointer_get (), 1);
MONO_PROFILER_RAISE (thread_stopped, (thread->tid));

/*
* This will signal async signal handlers that the thread has exited.
Expand Down Expand Up @@ -887,6 +884,8 @@ mono_thread_detach_internal (MonoInternalThread *thread)

mono_thread_info_unset_internal_thread_gchandle ((MonoThreadInfo*) thread->thread_info);

MONO_PROFILER_RAISE (thread_exited, (thread->tid));

/* Don't need to close the handle to this thread, even though we took a
* reference in mono_thread_attach (), because the GC will do it
* when the Thread object is finalised.
Expand Down Expand Up @@ -1276,9 +1275,7 @@ mono_thread_attach_full (MonoDomain *domain, gboolean force_attach)
if (mono_thread_attach_cb)
mono_thread_attach_cb (MONO_NATIVE_THREAD_ID_TO_UINT (tid), info->stack_end);

/* Can happen when we attach the profiler helper thread in order to heapshot. */
if (!mono_thread_info_current ()->tools_thread)
MONO_PROFILER_RAISE (thread_started, (MONO_NATIVE_THREAD_ID_TO_UINT (tid)));
MONO_PROFILER_RAISE (thread_started, (MONO_NATIVE_THREAD_ID_TO_UINT (tid)));

return thread;
}
Expand Down
36 changes: 22 additions & 14 deletions mono/profiler/log.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,12 @@ typedef struct {
int small_id;
} MonoProfilerThread;

// Default value in `profiler_tls` for new threads.
#define MONO_PROFILER_THREAD_ZERO ((MonoProfilerThread *) NULL)

// This is written to `profiler_tls` to indicate that a thread has stopped.
#define MONO_PROFILER_THREAD_DEAD ((MonoProfilerThread *) -1)

// Do not use these TLS macros directly unless you know what you're doing.

#ifdef HOST_WIN32
Expand Down Expand Up @@ -234,19 +240,19 @@ process_id (void)
#define ENTER_LOG(COUNTER, BUFFER, SIZE) \
do { \
MonoProfilerThread *thread__ = get_thread (); \
if (thread__->attached) \
buffer_lock (); \
g_assert (!thread__->busy && "Why are we trying to write a new event while already writing one?"); \
thread__->busy = TRUE; \
mono_atomic_inc_i32 ((COUNTER)); \
if (thread__->attached) \
buffer_lock (); \
LogBuffer *BUFFER = ensure_logbuf_unsafe (thread__, (SIZE))

#define EXIT_LOG_EXPLICIT(SEND) \
thread__->busy = FALSE; \
if ((SEND)) \
send_log_unsafe (TRUE); \
if (thread__->attached) \
buffer_unlock (); \
thread__->busy = FALSE; \
} while (0)

// Pass these to EXIT_LOG_EXPLICIT () for easier reading.
Expand Down Expand Up @@ -512,6 +518,8 @@ init_thread (gboolean add_to_lls)
{
MonoProfilerThread *thread = PROF_TLS_GET ();

g_assert (thread != MONO_PROFILER_THREAD_DEAD && "Why are we trying to resurrect a stopped thread?");
Copy link
Contributor

@kumpera kumpera Nov 28, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this keep happening frequently, we could return a dummy MonoProfilerThread instead.

Copy link
Contributor Author

@alexrp alexrp Nov 28, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should never happen, and indeed never did outside #5710, where it only happened because that PR introduced events that got fired after the thread_stopped event. I think an assertion is fine, we don't want a bug like this to go unnoticed. (Also, it already passed the stress tests in an earlier run of this PR, so I think we're good.)


/*
* Sometimes we may try to initialize a thread twice. One example is the
* main thread: We initialize it when setting up the profiler, but we will
Expand All @@ -523,14 +531,14 @@ init_thread (gboolean add_to_lls)
* These cases are harmless anyhow. Just return if we've already done the
* initialization work.
*/
if (thread)
if (thread != MONO_PROFILER_THREAD_ZERO)
return thread;

thread = g_malloc (sizeof (MonoProfilerThread));
thread->node.key = thread_id ();
thread->attached = add_to_lls;
thread->call_depth = 0;
thread->busy = 0;
thread->busy = FALSE;
thread->ended = FALSE;

init_buffer_state (thread);
Expand Down Expand Up @@ -559,7 +567,7 @@ deinit_thread (MonoProfilerThread *thread)
g_assert (!thread->attached && "Why are we manually freeing an attached thread?");

g_free (thread);
PROF_TLS_SET (NULL);
PROF_TLS_SET (MONO_PROFILER_THREAD_DEAD);
}

static MonoProfilerThread *
Expand Down Expand Up @@ -665,7 +673,7 @@ buffer_unlock (void)
gint32 state = mono_atomic_load_i32 (&log_profiler.buffer_lock_state);

// See the comment in buffer_lock ().
if (state == PROF_TLS_GET ()->small_id << 16)
if (state == get_thread ()->small_id << 16)
return;

g_assert (state && "Why are we decrementing a zero reader count?");
Expand Down Expand Up @@ -702,7 +710,7 @@ buffer_unlock_excl (void)
gint32 excl = state >> 16;

g_assert (excl && "Why is the exclusive lock not held?");
g_assert (excl == PROF_TLS_GET ()->small_id && "Why does another thread hold the exclusive lock?");
g_assert (excl == get_thread ()->small_id && "Why does another thread hold the exclusive lock?");
g_assert (!(state & 0xFFFF) && "Why are there readers when the exclusive lock is held?");

mono_atomic_store_i32 (&log_profiler.buffer_lock_state, 0);
Expand Down Expand Up @@ -1108,7 +1116,7 @@ dump_buffer_threadless (LogBuffer *buf)
static void
send_log_unsafe (gboolean if_needed)
{
MonoProfilerThread *thread = PROF_TLS_GET ();
MonoProfilerThread *thread = get_thread ();

if (!if_needed || (if_needed && thread->buffer->next)) {
if (!thread->attached)
Expand All @@ -1124,7 +1132,7 @@ send_log_unsafe (gboolean if_needed)
static void
sync_point_flush (void)
{
g_assert (mono_atomic_load_i32 (&log_profiler.buffer_lock_state) == PROF_TLS_GET ()->small_id << 16 && "Why don't we hold the exclusive lock?");
g_assert (mono_atomic_load_i32 (&log_profiler.buffer_lock_state) == get_thread ()->small_id << 16 && "Why don't we hold the exclusive lock?");

MONO_LLS_FOREACH_SAFE (&log_profiler.profiler_thread_list, MonoProfilerThread, thread) {
g_assert (thread->attached && "Why is a thread in the LLS not attached?");
Expand All @@ -1138,7 +1146,7 @@ sync_point_flush (void)
static void
sync_point_mark (MonoProfilerSyncPointType type)
{
g_assert (mono_atomic_load_i32 (&log_profiler.buffer_lock_state) == PROF_TLS_GET ()->small_id << 16 && "Why don't we hold the exclusive lock?");
g_assert (mono_atomic_load_i32 (&log_profiler.buffer_lock_state) == get_thread ()->small_id << 16 && "Why don't we hold the exclusive lock?");

ENTER_LOG (&sync_points_ctr, logbuffer,
EVENT_SIZE /* event */ +
Expand Down Expand Up @@ -2008,7 +2016,7 @@ thread_end (MonoProfiler *prof, uintptr_t tid)
thread->ended = TRUE;
remove_thread (thread);

PROF_TLS_SET (NULL);
PROF_TLS_SET (MONO_PROFILER_THREAD_DEAD);
}

static void
Expand Down Expand Up @@ -3965,7 +3973,7 @@ handle_writer_queue_entry (void)
g_ptr_array_free (entry->methods, TRUE);

if (wrote_methods) {
MonoProfilerThread *thread = PROF_TLS_GET ();
MonoProfilerThread *thread = get_thread ();

dump_buffer_threadless (thread->buffer);
init_buffer_state (thread);
Expand Down Expand Up @@ -4689,7 +4697,7 @@ mono_profiler_init_log (const char *desc)
mono_profiler_set_gc_event_callback (handle, gc_event);

mono_profiler_set_thread_started_callback (handle, thread_start);
mono_profiler_set_thread_stopped_callback (handle, thread_end);
mono_profiler_set_thread_exited_callback (handle, thread_end);
mono_profiler_set_thread_name_callback (handle, thread_name);

mono_profiler_set_domain_loaded_callback (handle, domain_loaded);
Expand Down