Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit b9155fd

Browse files
ylowylow
committed
dded thread local storage as well as deschedule/schedule functions to fiber
1 parent b978f06 commit b9155fd

5 files changed

Lines changed: 307 additions & 26 deletions

File tree

‎src/graphlab/parallel/fiber.cpp‎

Lines changed: 106 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ pthread_key_t fiber_group::tlskey;
1010
fiber_group::fiber_group(size_t nworkers, size_t stacksize)
1111
:nworkers(nworkers),
1212
stacksize(stacksize),
13-
stop_workers(false) {
13+
stop_workers(false),
14+
flsdeleter(NULL) {
1415
// initialize the thread local storage keys
1516
if (!tls_created) {
1617
pthread_key_create(&tlskey, fiber_group::tls_deleter);
@@ -24,7 +25,7 @@ fiber_group::fiber_group(size_t nworkers, size_t stacksize)
2425

2526
// launch the workers
2627
for (size_t i = 0;i < nworkers; ++i) {
27-
workers.launch(boost::bind(&fiber_group::worker_init, this));
28+
workers.launch(boost::bind(&fiber_group::worker_init, this, i));
2829
}
2930
}
3031

@@ -59,10 +60,12 @@ fiber_group::fiber* fiber_group::get_active_fiber() {
5960

6061

6162
void fiber_group::active_queue_insert(fiber_group::fiber* value) {
62-
value->next = NULL;
63-
active_tail->next = value;
64-
active_tail = value;
65-
++nactive;
63+
if (value->scheduleable) {
64+
value->next = NULL;
65+
active_tail->next = value;
66+
active_tail = value;
67+
++nactive;
68+
}
6669
// might want to handle the signalling mechanism here too
6770
}
6871

@@ -87,14 +90,15 @@ void fiber_group::exit() {
8790
}
8891
}
8992

90-
void fiber_group::worker_init() {
93+
void fiber_group::worker_init(size_t workerid) {
9194
// create a root context
9295
create_tls_ptr();
9396
// set up the tls structure
9497
tls* t = get_tls_ptr();
9598
t->prev_fiber = NULL;
9699
t->cur_fiber = NULL;
97100
t->garbage = NULL;
101+
t->worker_id = workerid;
98102
t->parent = this;
99103
active_lock.lock();
100104
while(!stop_workers) {
@@ -135,12 +139,16 @@ void fiber_group::trampoline(intptr_t _args) {
135139
size_t fiber_group::launch(void fn(void*), void* param) {
136140
// allocate a stack
137141
fiber* fib = new fiber;
142+
fib->parent = this;
138143
fib->stack = malloc(stacksize);
139144
fib->id = fiber_id_counter.inc();
140145
//VALGRIND_STACK_REGISTER(fib->stack, (char*)fib->stack + stacksize);
141-
fib->tls = NULL;
146+
fib->fls = NULL;
142147
fib->next = NULL;
148+
fib->deschedule_lock = NULL;
143149
fib->terminate = false;
150+
fib->descheduled = false;
151+
fib->scheduleable = true;
144152
// construct the initial context
145153
trampoline_args* args = new trampoline_args;
146154
args->fn = fn;
@@ -162,7 +170,7 @@ size_t fiber_group::launch(void fn(void*), void* param) {
162170
void fiber_group::yield_to(fiber* next_fib) {
163171
// the core scheduling logic
164172
tls* t = get_tls_ptr();
165-
/*
173+
/*
166174
if (next_fib) {
167175
printf("yield to: %ld\n", next_fib->id);
168176
if (t->cur_fiber) {
@@ -186,8 +194,9 @@ void fiber_group::yield_to(fiber* next_fib) {
186194
}
187195
} else {
188196
// ok. there isn't anything to schedule to
189-
// am I meant to be terminated?
190-
if (t->cur_fiber && t->cur_fiber->terminate) {
197+
// am I meant to be terminated? or descheduled?
198+
if (t->cur_fiber &&
199+
(t->cur_fiber->terminate || t->cur_fiber->descheduled) ) {
191200
// yup. killing current fiber
192201
// context switch back to basecontext which will
193202
// do the cleanup
@@ -213,25 +222,41 @@ void fiber_group::yield_to(fiber* next_fib) {
213222
}
214223

215224
void fiber_group::reschedule_fiber(fiber* fib) {
216-
if (!fib->terminate) {
225+
fib->lock.lock();
226+
if (!fib->terminate && !fib->descheduled) {
227+
fib->lock.unlock();
217228
// we reschedule it
218229
// Re-lock the queue
219-
active_lock.lock();
220230
//printf("Reinserting %ld\n", fib->id);
231+
active_lock.lock();
221232
active_queue_insert(fib);
222233
active_cond.signal();
223234
active_lock.unlock();
224-
} else {
235+
} else if (fib->descheduled) {
236+
// unflag descheduled and unset scheduleable
237+
fib->descheduled = false;
238+
fib->scheduleable = false;
239+
if (fib->deschedule_lock) pthread_mutex_unlock(fib->deschedule_lock);
240+
fib->deschedule_lock = NULL;
241+
//printf("Descheduling complete %ld\n", fib->id);
242+
fib->lock.unlock();
243+
} else if (fib->terminate) {
244+
fib->lock.unlock();
225245
// previous fiber is dead. destroy it
226246
free(fib->stack);
227247
//VALGRIND_STACK_DEREGISTER(fib->stack);
248+
// delete the fiber local storage if any
249+
if (fib->fls && flsdeleter) flsdeleter(fib->fls);
228250
delete fib;
229251
// if we are out of threads, signal the join
230252
if (fibers_active.dec() == 0) {
231253
join_lock.lock();
232254
join_cond.signal();
233255
join_lock.unlock();
234256
}
257+
} else {
258+
// impossible condition
259+
assert(false);
235260
}
236261
}
237262

@@ -254,4 +279,71 @@ void fiber_group::join() {
254279
join_lock.unlock();
255280
}
256281

282+
size_t fiber_group::get_tid() {
283+
return reinterpret_cast<size_t>(get_tls_ptr()->cur_fiber);
284+
}
285+
286+
void fiber_group::deschedule_self(pthread_mutex_t* lock) {
287+
fiber* fib = get_tls_ptr()->cur_fiber;
288+
fib->lock.lock();
289+
assert(fib->descheduled == false);
290+
assert(fib->scheduleable == true);
291+
fib->deschedule_lock = lock;
292+
fib->descheduled = true;
293+
//printf("Descheduling requested %ld\n", fib->id);
294+
fib->lock.unlock();
295+
yield();
296+
}
297+
298+
size_t fiber_group::get_worker_id() {
299+
fiber_group::tls* tls = get_tls_ptr();
300+
return tls->worker_id;
301+
}
302+
303+
void fiber_group::schedule_tid(size_t tid) {
304+
fiber* fib = reinterpret_cast<fiber*>(tid);
305+
fib->lock.lock();
306+
// we MUST get here only after the thread was completely descheduled
307+
// or no deschedule operation has happened yet.
308+
assert(fib->descheduled == false);
309+
fib->descheduled = false;
310+
if (fib->scheduleable == false) {
311+
// if this thread was descheduled completely. Reschedule it.
312+
//printf("Scheduling requested %ld\n", fib->id);
313+
fib->scheduleable = true;
314+
fib->lock.unlock();
315+
fib->parent->reschedule_fiber(fib);
316+
} else {
317+
//printf("Scheduling requested of running thread %ld\n", fib->id);
318+
fib->lock.unlock();
319+
}
320+
}
321+
322+
323+
void fiber_group::set_tls_deleter(void (*deleter)(void*)) {
324+
flsdeleter = deleter;
325+
}
326+
327+
void* fiber_group::get_tls() {
328+
fiber_group::tls* f = get_tls_ptr();
329+
if (f != NULL) {
330+
return f->cur_fiber->fls;
331+
} else {
332+
// cannot get TLS of a non-fiber
333+
assert(false);
334+
return NULL;
335+
}
336+
}
337+
338+
void fiber_group::set_tls(void* tls) {
339+
fiber_group::tls* f = get_tls_ptr();
340+
if (f != NULL) {
341+
f->cur_fiber->fls = tls;
342+
} else {
343+
// cannot get TLS of a non-fiber
344+
assert(false);
345+
}
346+
}
347+
348+
257349
}

‎src/graphlab/parallel/fiber.hpp‎

Lines changed: 117 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,30 @@ namespace graphlab {
1010
class fiber_group {
1111
public:
1212
struct fiber {
13-
boost::context::fcontext_t* context;
13+
simple_spinlock lock;
1414
fiber_group* parent;
15+
boost::context::fcontext_t* context;
1516
void* stack;
1617
size_t id;
17-
void* tls;
18+
void* fls; // fiber local storage
1819
fiber* next;
1920
intptr_t initial_trampoline_args;
20-
bool terminate;
21+
pthread_mutex_t* deschedule_lock; // if descheduled is set, we will
22+
// atomically deschedule and unlock
23+
// this mutex
24+
bool descheduled; // flag. set if this fiber is to be descheduled.
25+
// This is a temporary flag, and is only used to notify
26+
// the context switch to deschedule this thread.
27+
// lock must be acquired for this to be modified
28+
29+
bool terminate; // flag. set if this fiber is to be destroyed.
30+
// This is a temporary flag, and is only used to notify
31+
// the context switch to destroy this thread.
32+
33+
bool scheduleable; // Managed by the queue management routines.
34+
// Set if the fiber is inside the scheduling queue
35+
// or is running in a thread.
36+
// lock must be acquired for this to be modified.
2137
};
2238

2339

@@ -55,27 +71,30 @@ class fiber_group {
5571
fiber* prev_fiber; // the fiber we context switch from
5672
fiber* cur_fiber; // the fiber we are context switching to
5773
fiber* garbage;
74+
size_t worker_id;
5875
boost::context::fcontext_t base_context;
5976
};
60-
static pthread_key_t tlskey;
77+
static pthread_key_t tlskey; // points to the tls structure above
6178
static void tls_deleter(void* tls);
6279

6380
static void create_tls_ptr();
6481
static tls* get_tls_ptr();
6582
static fiber* get_active_fiber();
6683
static void set_active_fiber(fiber*);
6784

68-
void worker_init();
85+
void worker_init(size_t workerid);
6986

7087
void reschedule_fiber(fiber* pfib);
7188
void yield_to(fiber* next_fib);
7289
static void trampoline(intptr_t _args);
7390

91+
void (*flsdeleter)(void*);
7492
public:
7593
/// creates a group of fibers using a certain number of worker threads.
7694
fiber_group(size_t nworkers, size_t stacksize);
7795

7896
~fiber_group();
97+
7998
/** the basic launch function
8099
* More advanced ones (for instance using boost::function)
81100
* can be built on top of this easily.
@@ -84,9 +103,102 @@ class fiber_group {
84103
*/
85104
size_t launch(void fn(void*), void* param);
86105

106+
/**
107+
* Waits for all functions to join
108+
*/
87109
void join();
110+
111+
inline size_t total_threads_created() {
112+
return fiber_id_counter.value;
113+
}
114+
/**
115+
* Sets the TLS deletion function. The deletion function will be called
116+
* on every non-NULL TLS value.
117+
*/
118+
void set_tls_deleter(void (*deleter)(void*));
119+
/**
120+
* Gets the TLS value. Defaults to NULL.
121+
* Note that this function will only work within a fiber.
122+
*/
123+
static void* get_tls();
124+
/**
125+
* Sets the TLS value.
126+
* Note that this function will only work within a fiber.
127+
* If the value is not NULL, and the deletion function is set by
128+
* set_tls_deleter(), the deleter will be called on the value on
129+
* fiber termination.
130+
*/
131+
static void set_tls(void* value);
132+
/**
133+
* Kills the current fiber.
134+
* Note that this function will only work within a fiber.
135+
*/
88136
static void exit();
137+
/**
138+
* Yields to another fiber.
139+
* Note that this function will only work within a fiber.
140+
*/
89141
static void yield();
142+
/**
143+
* Returns the current fiber handle.
144+
* Note that fiber handles are not sequential, and are really a
145+
* pointer to an internal datastructure.
146+
* The fiber handle will never be 0.
147+
* This function will only work within a fiber.
148+
*/
149+
static size_t get_tid();
150+
151+
/**
152+
* Returns the worker managing the current fiber.
153+
* This function will only work within a fiber.
154+
* Worker IDs are sequential.
155+
*/
156+
static size_t get_worker_id();
157+
158+
159+
/**
160+
* Atomically deschedules the current thread and unlocks the mutex.
161+
*
162+
* deschedule_self() and schedule_tid() must be managed carefully
163+
* to avoid race conditions. i.e. schedule_tid() happending before
164+
* deschedule_self().
165+
*
166+
* To support this correctly, the descheduling must be paired together
167+
* with a mutex.
168+
*
169+
* For instance, to use this to implement a promise.
170+
* \code
171+
* // descheduling fiber
172+
* pthread_mutex_lock(&lock);
173+
* if ( ... promise not ready ...) {
174+
* deschedule_self(&lock);
175+
* } else {
176+
* pthread_mutex_unlock(&lock);
177+
* }
178+
* ... use the promise ...
179+
* \endcode
180+
*
181+
*
182+
* The promise execution thread then must do the following
183+
* \code
184+
* ... tid contains the fiber ID to wake when promise is done
185+
* ... compute promise...
186+
* pthread_mutex_lock(&lock); // same lock as above
187+
* ... set promise completion...
188+
* schedule_tid(tid); // wake up the fiber
189+
* pthread_mutex_unlock(&lock);
190+
* \endcode
191+
*/
192+
static void deschedule_self(pthread_mutex_t* lock);
193+
194+
/**
195+
* Schedules a fiber for execution.
196+
* If this fiber was previously descheduled by
197+
* deschedule_self(), the fiber is scheduled for execution.
198+
* Otherwise, nothing happens. Some care must be taken to avoid race
199+
* conditions. See the deschedule_self() function for details.
200+
*/
201+
static void schedule_tid(size_t tid);
90202
};
91203

92204
}

0 commit comments

Comments
 (0)