Deadlock when running subflows in a pipeline #402
Description
When running subflows from within a pipe in a pipeline, I'm getting random deadlocks. I've simplified my code down to the following example which will sometimes run to completion within a few seconds but other times get stuck and stop making progress:
#include <thread>
#include <taskflow/taskflow.hpp>
#include <taskflow/algorithm/pipeline.hpp>
int main() {
tf::Taskflow taskflow("pipeline-test");
tf::Executor executor;
size_t num_lines = std::thread::hardware_concurrency();
tf::Pipeline pl(
num_lines,
tf::Pipe{tf::PipeType::SERIAL, [](tf::Pipeflow& pf) {
if (pf.token() == 1'000'000) {
pf.stop();
}
}},
tf::Pipe{tf::PipeType::PARALLEL, [](tf::Pipeflow& pf, tf::Runtime& rt) {
constexpr size_t num_subtasks = 2;
rt.run([](tf::Subflow& sf) {
for (size_t i = 0; i < num_subtasks; ++i) {
sf.emplace([]() {
std::this_thread::sleep_for(std::chrono::nanoseconds(1));
});
}
});
}}
);
taskflow.composed_of(pl).name("pipeline");
executor.run(taskflow).wait();
}I'm using GCC 11.3.1 on Fedora Linux 35, with an AMD Ryzen 9 5900X (12 physical cores, 24 logical cores), and using taskflow v3.3.0.
If I debug the deadlock I see 25 threads running and stacks that look like this:
Thread-1:
syscall 0x00007ffff7a86ecd
std::__atomic_futex_unsigned_base::_M_futex_wait_until(unsigned int*, unsigned int, bool, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000000000l> >) 0x00007ffff7e1be33
std::__atomic_futex_unsigned::_M_load_and_test_until atomic_futex.h:110
std::__atomic_futex_unsigned::_M_load_and_test atomic_futex.h:159
std::__atomic_futex_unsigned::_M_load_when_equal atomic_futex.h:213
std::__future_base::_State_baseV2::wait future:336
std::__basic_future::wait future:694
main taskflow_test.cc:33
__libc_start_call_main 0x00007ffff7a87f20
__libc_start_main_impl 0x00007ffff7a87fd0
_start 0x0000000000405615
Most other threads:
__futex_abstimed_wait_common 0x00007ffff7ae43fa
pthread_cond_wait@@GLIBC_2.3.2 0x00007ffff7ae6bf0
std::condition_variable::wait(std::unique_lock<std::mutex>&) 0x00007ffff7e186e0
tf::Notifier::_park notifier.hpp:243
tf::Notifier::commit_wait notifier.hpp:134
tf::Executor::_wait_for_task executor.hpp:1027
<lambda#1>::operator()(tf::Worker &, std::mutex &, std::condition_variable &, unsigned long &) const executor.hpp:863
std::__invoke_impl<…>(std::__invoke_other, <lambda#1> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&) invoke.h:61
std::__invoke<…>(<lambda#1> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&) invoke.h:96
std::thread::_Invoker::_M_invoke<…>(std::_Index_tuple<…>) std_thread.h:253
std::thread::_Invoker::operator()() std_thread.h:260
std::thread::_State_impl::_M_run() std_thread.h:211
execute_native_thread_routine 0x00007ffff7e1e5a4
start_thread 0x00007ffff7ae785a
clone3 0x00007ffff7a874c0
Thread-11:
sched_yield 0x00007ffff7a8672b
__gthread_yield gthr-default.h:693
std::this_thread::yield std_thread.h:323
tf::Executor::_consume_task executor.hpp:900
tf::Executor::_join_dynamic_task_internal executor.hpp:1502
tf::Runtime::run<…>(struct {...} &&) executor.hpp:1967
operator() taskflow_test.cc:22
operator()<…>(tf::Pipe<…> &) const pipeline.hpp:418
std::__invoke_impl<void, tf::Pipeline<tf::Pipe<main()::<lambda(tf::Pipeflow&)> >, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> > >::_on_pipe(tf::Pipeflow&, tf::Runtime&)::<lambda(auto:49&&)>&, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> >&>(std::__invoke_other, struct {...} &) invoke.h:61
std::__invoke<tf::Pipeline<tf::Pipe<main()::<lambda(tf::Pipeflow&)> >, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> > >::_on_pipe(tf::Pipeflow&, tf::Runtime&)::<lambda(auto:49&&)>&, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> >&>(struct {...} &) invoke.h:96
std::invoke<tf::Pipeline<tf::Pipe<main()::<lambda(tf::Pipeflow&)> >, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> > >::_on_pipe(tf::Pipeflow&, tf::Runtime&)::<lambda(auto:49&&)>&, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> >&>(struct {...} &) functional:97
tf::visit_tuple<tf::Pipeline<tf::Pipe<main()::<lambda(tf::Pipeflow&)> >, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> > >::_on_pipe(tf::Pipeflow&, tf::Runtime&)::<lambda(auto:49&&)>, std::tuple<tf::Pipe<main()::<lambda(tf::Pipeflow&)> >, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> > >, 1>(struct {...}, std::tuple<tf::Pipe<main()::<lambda(tf::Pipeflow&)> >, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> > > &, size_t) traits.hpp:194
tf::visit_tuple<tf::Pipeline<tf::Pipe<main()::<lambda(tf::Pipeflow&)> >, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> > >::_on_pipe(tf::Pipeflow&, tf::Runtime&)::<lambda(auto:49&&)>, std::tuple<tf::Pipe<main()::<lambda(tf::Pipeflow&)> >, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> > > >(struct {...}, std::tuple<tf::Pipe<main()::<lambda(tf::Pipeflow&)> >, tf::Pipe<main()::<lambda(tf::Pipeflow&, tf::Runtime&)> > > &, size_t) traits.hpp:198
tf::Pipeline::_on_pipe(tf::Pipeflow &, tf::Runtime &) pipeline.hpp:412
operator() pipeline.hpp:461
std::__invoke_impl<…>(std::__invoke_other, struct {...} &) invoke.h:61
std::__invoke_r<…>(struct {...} &) invoke.h:111
std::_Function_handler::_M_invoke(const std::_Any_data &, tf::Runtime &) std_function.h:290
std::function::operator()(tf::Runtime &) const std_function.h:590
tf::Executor::_invoke_runtime_task executor.hpp:1564
tf::Executor::_invoke executor.hpp:1250
tf::Executor::_exploit_task executor.hpp:955
<lambda#1>::operator()(tf::Worker &, std::mutex &, std::condition_variable &, unsigned long &) const executor.hpp:860
std::__invoke_impl<…>(std::__invoke_other, <lambda#1> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&) invoke.h:61
std::__invoke<…>(<lambda#1> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&) invoke.h:96
std::thread::_Invoker::_M_invoke<…>(std::_Index_tuple<…>) std_thread.h:253
std::thread::_Invoker::operator()() std_thread.h:260
std::thread::_State_impl::_M_run() std_thread.h:211
execute_native_thread_routine 0x00007ffff7e1e5a4
start_thread 0x00007ffff7ae785a
clone3 0x00007ffff7a874c0
Thread-21:
sched_yield 0x00007ffff7a8672b
__gthread_yield gthr-default.h:693
std::this_thread::yield std_thread.h:323
tf::Executor::_explore_task executor.hpp:934
tf::Executor::_wait_for_task executor.hpp:974
<lambda#1>::operator()(tf::Worker &, std::mutex &, std::condition_variable &, unsigned long &) const executor.hpp:863
std::__invoke_impl<…>(std::__invoke_other, <lambda#1> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&) invoke.h:61
std::__invoke<…>(<lambda#1> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&) invoke.h:96
std::thread::_Invoker::_M_invoke<…>(std::_Index_tuple<…>) std_thread.h:253
std::thread::_Invoker::operator()() std_thread.h:260
std::thread::_State_impl::_M_run() std_thread.h:211
execute_native_thread_routine 0x00007ffff7e1e5a4
start_thread 0x00007ffff7ae785a
clone3 0x00007ffff7a874c0
Thread-25:
sched_yield 0x00007ffff7a8672b
__gthread_yield gthr-default.h:693
std::this_thread::yield std_thread.h:323
tf::Executor::_consume_task executor.hpp:900
tf::Executor::_join_dynamic_task_internal executor.hpp:1502
tf::Executor::_invoke_module_task executor.hpp:1540
tf::Executor::_invoke executor.hpp:1216
tf::Executor::_exploit_task executor.hpp:955
<lambda#1>::operator()(tf::Worker &, std::mutex &, std::condition_variable &, unsigned long &) const executor.hpp:860
std::__invoke_impl<…>(std::__invoke_other, <lambda#1> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&) invoke.h:61
std::__invoke<…>(<lambda#1> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&, std::reference_wrapper<…> &&) invoke.h:96
std::thread::_Invoker::_M_invoke<…>(std::_Index_tuple<…>) std_thread.h:253
std::thread::_Invoker::operator()() std_thread.h:260
std::thread::_State_impl::_M_run() std_thread.h:211
execute_native_thread_routine 0x00007ffff7e1e5a4
start_thread 0x00007ffff7ae785a
clone3 0x00007ffff7a874c0
Debugging multiple runs that deadlock shows the same pattern of stack traces each time, but with different thread numbers corresponding to the last three stacks.