From 6b52ebaa18e72207ab6b5d79a95ef995e7c94d59 Mon Sep 17 00:00:00 2001 From: Bright Chen Date: Thu, 19 Dec 2024 23:56:16 +0800 Subject: [PATCH] Opt signal trace --- README_cn.md | 1 + docs/cn/bthread_tracer.md | 12 +- src/brpc/builtin/bthreads_service.cpp | 2 +- src/brpc/input_messenger.cpp | 3 +- src/brpc/reloadable_flags.cpp | 26 +- src/brpc/reloadable_flags.h | 47 +-- src/brpc/shared_object.h | 46 +-- src/bthread/task_control.cpp | 2 +- src/bthread/task_group.cpp | 13 +- src/bthread/task_tracer.cpp | 471 +++++++++++++++----------- src/bthread/task_tracer.h | 79 +++-- src/butil/memory/scope_guard.h | 6 +- src/butil/reloadable_flags.h | 75 ++++ src/butil/shared_object.h | 73 ++++ test/bthread_unittest.cpp | 15 +- 15 files changed, 536 insertions(+), 335 deletions(-) create mode 100644 src/butil/reloadable_flags.h create mode 100644 src/butil/shared_object.h diff --git a/README_cn.md b/README_cn.md index 66f194e81d..bed6e8437f 100644 --- a/README_cn.md +++ b/README_cn.md @@ -38,6 +38,7 @@ * [bthread or not](docs/cn/bthread_or_not.md) * [thread-local](docs/cn/thread_local.md) * [Execution Queue](docs/cn/execution_queue.md) + * [bthread tracer](docs/cn/bthread_tracer.md) * Client * [基础功能](docs/cn/client.md) * [错误码](docs/cn/error_code.md) diff --git a/docs/cn/bthread_tracer.md b/docs/cn/bthread_tracer.md index 5eb2ce86ee..7758bee184 100644 --- a/docs/cn/bthread_tracer.md +++ b/docs/cn/bthread_tracer.md @@ -63,7 +63,17 @@ jump_stack是bthread挂起或者运行的必经之路，也是STB的拦截点。 3. 访问服务的内置服务：`http://ip:port/bthreads/?st=1`或者代码里调用`bthread::stack_trace()`函数。 4. 如果希望追踪pthread的调用栈，在对应pthread上调用`bthread::init_for_pthread_stack_trace()`函数获取一个伪bthread_t，然后使用步骤3即可获取pthread调用栈。 +下面是追踪bthread调用栈的输出示例： +```shell +#0 0x00007fdbbed500b5 __clock_gettime_2 +#1 0x000000000041f2b6 butil::cpuwide_time_ns() +#2 0x000000000041f289 butil::cpuwide_time_us() +#3 0x000000000041f1b9 butil::EveryManyUS::operator bool() +#4 0x0000000000413289 (anonymous namespace)::spin_and_log() +#5 0x00007fdbbfa58dc0 bthread::TaskGroup::task_runner() +``` + # 相关flag -- `enable_fast_unwind`：是否启用快速回溯功能，默认为true。大多数情况下，不需要关闭快速回溯功能。除非你关注的调用栈函数名转换失败，显示为``，则可以尝试关闭快速回溯功能，但这会导致性能下降。以包含30帧的调用栈举例，快速回溯只需要400~500us，而关闭快速回溯则需要4ms左右，性能下降了近10倍。 +- `enable_fast_unwind`：是否启用快速回溯功能，默认为true。大多数情况下，不需要关闭快速回溯功能。除非你关注的调用栈函数名转换失败，显示为``，则可以尝试关闭快速回溯功能，但这会导致性能下降。以包含30帧的调用栈举例，快速回溯基本上在200us以内就可以完成，而关闭快速回溯则需要4ms左右，性能下降了近20倍。 - `signal_trace_timeout_ms`：信号追踪模式的超时时间，默认为50ms。虽然libunwind文档显示回溯功能是异步信号安全的，但是[gpertools社区发现libunwind在某些情况下会死锁](https://github.com/gperftools/gperftools/issues/775)，所以TaskTracer会设置了超时时间，超时后会放弃回溯，打破死锁。 \ No newline at end of file diff --git a/src/brpc/builtin/bthreads_service.cpp b/src/brpc/builtin/bthreads_service.cpp index d95cb8f144..7676f65600 100644 --- a/src/brpc/builtin/bthreads_service.cpp +++ b/src/brpc/builtin/bthreads_service.cpp @@ -23,7 +23,7 @@ #include "brpc/builtin/bthreads_service.h" namespace bthread { -void print_task(std::ostream& os, bthread_t tid); +extern void print_task(std::ostream& os, bthread_t tid); } diff --git a/src/brpc/input_messenger.cpp b/src/brpc/input_messenger.cpp index d6e1c35670..b28c704415 100644 --- a/src/brpc/input_messenger.cpp +++ b/src/brpc/input_messenger.cpp @@ -287,8 +287,7 @@ int InputMessenger::ProcessNewMessage( // This unique_ptr prevents msg to be lost before transfering // ownership to last_msg DestroyingPtr msg(pr.message()); - QueueMessage(last_msg.release(), &num_bthread_created, - m->_keytable_pool); + QueueMessage(last_msg.release(), &num_bthread_created, m->_keytable_pool); if (_handlers[index].process == NULL) { LOG(ERROR) << "process of index=" << index << " is NULL"; continue; diff --git a/src/brpc/reloadable_flags.cpp b/src/brpc/reloadable_flags.cpp index 5f5ca026f8..958dc057ae 100644 --- a/src/brpc/reloadable_flags.cpp +++ b/src/brpc/reloadable_flags.cpp @@ -15,10 +15,6 @@ // specific language governing permissions and limitations // under the License. - -#include // write, _exit -#include -#include "butil/macros.h" #include "brpc/reloadable_flags.h" namespace brpc { @@ -62,37 +58,25 @@ bool NonNegativeInteger(const char*, int64_t val) { return val >= 0; } -template -static bool RegisterFlagValidatorOrDieImpl( - const T* flag, bool (*validate_fn)(const char*, T val)) { - if (GFLAGS_NS::RegisterFlagValidator(flag, validate_fn)) { - return true; - } - // Error printed by gflags does not have newline. Add one to it. - char newline = '\n'; - butil::ignore_result(write(2, &newline, 1)); - _exit(1); -} - bool RegisterFlagValidatorOrDie(const bool* flag, bool (*validate_fn)(const char*, bool)) { - return RegisterFlagValidatorOrDieImpl(flag, validate_fn); + return butil::RegisterFlagValidatorOrDieImpl(flag, validate_fn); } bool RegisterFlagValidatorOrDie(const int32_t* flag, bool (*validate_fn)(const char*, int32_t)) { - return RegisterFlagValidatorOrDieImpl(flag, validate_fn); + return butil::RegisterFlagValidatorOrDieImpl(flag, validate_fn); } bool RegisterFlagValidatorOrDie(const int64_t* flag, bool (*validate_fn)(const char*, int64_t)) { - return RegisterFlagValidatorOrDieImpl(flag, validate_fn); + return butil::RegisterFlagValidatorOrDieImpl(flag, validate_fn); } bool RegisterFlagValidatorOrDie(const uint64_t* flag, bool (*validate_fn)(const char*, uint64_t)) { - return RegisterFlagValidatorOrDieImpl(flag, validate_fn); + return butil::RegisterFlagValidatorOrDieImpl(flag, validate_fn); } bool RegisterFlagValidatorOrDie(const double* flag, bool (*validate_fn)(const char*, double)) { - return RegisterFlagValidatorOrDieImpl(flag, validate_fn); + return butil::RegisterFlagValidatorOrDieImpl(flag, validate_fn); } } // namespace brpc diff --git a/src/brpc/reloadable_flags.h b/src/brpc/reloadable_flags.h index c451bde850..dabd86b5a8 100644 --- a/src/brpc/reloadable_flags.h +++ b/src/brpc/reloadable_flags.h @@ -19,28 +19,7 @@ #ifndef BRPC_RELOADABLE_FLAGS_H #define BRPC_RELOADABLE_FLAGS_H -// To brpc developers: This is a header included by user, don't depend -// on internal structures, use opaque pointers instead. - -#include - -// Register an always-true valiator to a gflag so that the gflag is treated as -// reloadable by brpc. If a validator exists, abort the program. -// You should call this macro within global scope. for example: -// -// DEFINE_int32(foo, 0, "blah blah"); -// BRPC_VALIDATE_GFLAG(foo, brpc::PassValidate); -// -// This macro does not work for string-flags because they're thread-unsafe to -// modify directly. To emphasize this, you have to write the validator by -// yourself and use GFLAGS_NS::GetCommandLineOption() to acess the flag. -#define BRPC_VALIDATE_GFLAG(flag, validate_fn) \ - namespace brpc_flags {} \ - const int register_FLAGS_ ## flag ## _dummy \ - __attribute__((__unused__)) = \ - ::brpc::RegisterFlagValidatorOrDie( \ - &FLAGS_##flag, (validate_fn)) - +#include "butil/reloadable_flags.h" namespace brpc { @@ -59,18 +38,18 @@ extern bool PositiveInteger(const char*, uint64_t); extern bool NonNegativeInteger(const char*, int32_t); extern bool NonNegativeInteger(const char*, int64_t); -extern bool RegisterFlagValidatorOrDie(const bool* flag, - bool (*validate_fn)(const char*, bool)); -extern bool RegisterFlagValidatorOrDie(const int32_t* flag, - bool (*validate_fn)(const char*, int32_t)); -extern bool RegisterFlagValidatorOrDie(const uint32_t* flag, - bool (*validate_fn)(const char*, uint32_t)); -extern bool RegisterFlagValidatorOrDie(const int64_t* flag, - bool (*validate_fn)(const char*, int64_t)); -extern bool RegisterFlagValidatorOrDie(const uint64_t* flag, - bool (*validate_fn)(const char*, uint64_t)); -extern bool RegisterFlagValidatorOrDie(const double* flag, - bool (*validate_fn)(const char*, double)); +extern bool RegisterFlagValidatorOrDie( + const bool* flag, bool (*validate_fn)(const char*, bool)); +extern bool RegisterFlagValidatorOrDie( + const int32_t* flag, bool (*validate_fn)(const char*, int32_t)); +extern bool RegisterFlagValidatorOrDie( + const uint32_t* flag, bool (*validate_fn)(const char*, uint32_t)); +extern bool RegisterFlagValidatorOrDie( + const int64_t* flag, bool (*validate_fn)(const char*, int64_t)); +extern bool RegisterFlagValidatorOrDie( + const uint64_t* flag, bool (*validate_fn)(const char*, uint64_t)); +extern bool RegisterFlagValidatorOrDie( + const double* flag, bool (*validate_fn)(const char*, double)); } // namespace brpc diff --git a/src/brpc/shared_object.h b/src/brpc/shared_object.h index d8ff9aaedc..296bdea325 100644 --- a/src/brpc/shared_object.h +++ b/src/brpc/shared_object.h @@ -19,53 +19,11 @@ #ifndef BRPC_SHARED_OBJECT_H #define BRPC_SHARED_OBJECT_H -#include "butil/intrusive_ptr.hpp" // butil::intrusive_ptr -#include "butil/atomicops.h" - +#include "butil/shared_object.h" namespace brpc { -// Inherit this class to be intrusively shared. Comparing to shared_ptr, -// intrusive_ptr saves one malloc (for shared_count) and gets better cache -// locality when the ref/deref are frequent, in the cost of lack of weak_ptr -// and worse interface. -class SharedObject { -friend void intrusive_ptr_add_ref(SharedObject*); -friend void intrusive_ptr_release(SharedObject*); - -public: - SharedObject() : _nref(0) { } - int ref_count() const { return _nref.load(butil::memory_order_relaxed); } - - // Add ref and returns the ref_count seen before added. - // The effect is basically same as butil::intrusive_ptr(obj).detach() - // except that the latter one does not return the seen ref_count which is - // useful in some scenarios. - int AddRefManually() - { return _nref.fetch_add(1, butil::memory_order_relaxed); } - - // Remove one ref, if the ref_count hit zero, delete this object. - // Same as butil::intrusive_ptr(obj, false).reset(NULL) - void RemoveRefManually() { - if (_nref.fetch_sub(1, butil::memory_order_release) == 1) { - butil::atomic_thread_fence(butil::memory_order_acquire); - delete this; - } - } - -protected: - virtual ~SharedObject() { } -private: - butil::atomic _nref; -}; - -inline void intrusive_ptr_add_ref(SharedObject* obj) { - obj->AddRefManually(); -} - -inline void intrusive_ptr_release(SharedObject* obj) { - obj->RemoveRefManually(); -} +using butil::SharedObject; } // namespace brpc diff --git a/src/bthread/task_control.cpp b/src/bthread/task_control.cpp index 96922b48e4..55ed1f2e42 100644 --- a/src/bthread/task_control.cpp +++ b/src/bthread/task_control.cpp @@ -216,7 +216,7 @@ int TaskControl::init(int concurrency) { } #ifdef BRPC_BTHREAD_TRACER - if (_task_tracer.Init() != 0) { + if (!_task_tracer.Init()) { LOG(ERROR) << "Fail to init TaskTracer"; return -1; } diff --git a/src/bthread/task_group.cpp b/src/bthread/task_group.cpp index 23333d4388..243394d177 100644 --- a/src/bthread/task_group.cpp +++ b/src/bthread/task_group.cpp @@ -660,8 +660,13 @@ void TaskGroup::sched_to(TaskGroup** pg, TaskMeta* next_meta) { CHECK(cur_meta->stack == g->_main_stack); } #endif + } /* else because of ending_sched(including pthread_task->pthread_task). */ +#ifdef BRPC_BTHREAD_TRACER + else { + // _cur_meta: TASK_STATUS_FIRST_READY -> TASK_STATUS_RUNNING. + TaskTracer::set_running_status(g->tid(), g->_cur_meta); } - // else because of ending_sched(including pthread_task->pthread_task) +#endif // BRPC_BTHREAD_TRACER } else { LOG(FATAL) << "bthread=" << g->current_tid() << " sched_to itself!"; } @@ -1013,13 +1018,15 @@ void print_task(std::ostream& os, bthread_t tid) { << "}\nhas_tls=" << has_tls << "\nuptime_ns=" << butil::cpuwide_time_ns() - cpuwide_start_ns << "\ncputime_ns=" << stat.cputime_ns - << "\nnswitch=" << stat.nswitch + << "\nnswitch=" << stat.nswitch #ifdef BRPC_BTHREAD_TRACER << "\nstatus=" << status << "\ntraced=" << traced << "\nworker_tid=" << worker_tid; -#endif // BRPC_BTHREAD_TRACER +#else ; + (void)status;(void)traced;(void)worker_tid; +#endif // BRPC_BTHREAD_TRACER } } diff --git a/src/bthread/task_tracer.cpp b/src/bthread/task_tracer.cpp index 2d8e509d6d..dccb0c22e4 100644 --- a/src/bthread/task_tracer.cpp +++ b/src/bthread/task_tracer.cpp @@ -18,8 +18,12 @@ #ifdef BRPC_BTHREAD_TRACER #include "bthread/task_tracer.h" +#include +#include +#include #include "butil/debug/stack_trace.h" #include "butil/memory/scope_guard.h" +#include "butil/reloadable_flags.h" #include "bthread/task_group.h" #include "bthread/processor.h" @@ -27,23 +31,132 @@ namespace bthread { DEFINE_bool(enable_fast_unwind, true, "Whether to enable fast unwind"); DEFINE_uint32(signal_trace_timeout_ms, 50, "Timeout for signal trace in ms"); +BRPC_VALIDATE_GFLAG(signal_trace_timeout_ms, butil::PositiveInteger); extern BAIDU_THREAD_LOCAL TaskMeta* pthread_fake_meta; -int TaskTracer::Init() { - if (RegisterSignalHandler() != 0) { - return -1; +TaskTracer::SignalSync::~SignalSync() { + if (_pipe_init) { + close(pipe_fds[0]); + close(pipe_fds[1]); } + + if (_sem_init) { + sem_destroy(&sem); + } +} + +bool TaskTracer::SignalSync::Init() { + if (pipe(pipe_fds) != 0) { + PLOG(ERROR) << "Fail to pipe"; + return false; + } + if (butil::make_non_blocking(pipe_fds[0]) != 0) { + PLOG(ERROR) << "Fail to make_non_blocking"; + return false; + } + if (butil::make_non_blocking(pipe_fds[1]) != 0) { + PLOG(ERROR) << "Fail to make_non_blocking"; + return false; + } + _pipe_init = true; + + if (sem_init(&sem, 0, 0) != 0) { + PLOG(ERROR) << "Fail to sem_init"; + return false; + } + _sem_init = true; + + return true; +} + +std::string TaskTracer::Result::OutputToString() { + std::string str; + if (err_count > 0 || frame_count > 0) { + str.reserve(1024); + } + if (frame_count > 0) { + if (fast_unwind) { + butil::debug::StackTrace stack_trace((void**)&ips, frame_count); + stack_trace.OutputToString(str); + } else { + for (size_t i = 0; i < frame_count; ++i) { + butil::string_appendf(&str, "#%zu 0x%016lx ", i, ips[i]); + if (mangled[i][0] == '\0') { + str.append(""); + } else { + str.append(butil::demangle(mangled[i])); + } + if (i + 1 < frame_count) { + str.push_back('\n'); + } + } + } + } else { + str.append("No frame"); + } + + if (err_count > 0) { + str.append("\nError message:\n"); + } + for (size_t i = 0; i < err_count; ++i) { + str.append(err_msg[i]); + if (i + 1 < err_count) { + str.push_back('\n'); + } + } + + return str; +} + +void TaskTracer::Result::OutputToStream(std::ostream& os) { + if (frame_count > 0) { + if (fast_unwind) { + butil::debug::StackTrace stack_trace((void**)&ips, frame_count); + stack_trace.OutputToStream(&os); + } else { + for (size_t i = 0; i < frame_count; ++i) { + os << "# " << i << " 0x" << std::hex << ips[i] << std::dec << " "; + if (mangled[i][0] == '\0') { + os << ""; + } else { + os << butil::demangle(mangled[i]); + } + if (i + 1 < frame_count) { + os << '\n'; + } + } + } + } else { + os << "No frame"; + } + + if (err_count == 0) { + return; + } + + os << "\nError message:\n"; + for (size_t i = 0; i < err_count; ++i) { + os << err_msg[i]; + if (i + 1 < err_count) { + os << '\n'; + } + } +} + +bool TaskTracer::Init() { if (_trace_time.expose("bthread_trace_time") != 0) { - return -1; + return false; } - if (_unwind_time.expose("bthread_unwind_time") != 0) { - return -1; + if (!RegisterSignalHandler()) { + return false; } - if (_signal_handler_time.expose("bthread_signal_handler_time") != 0) { - return -1; + // Warm up the libunwind. + unw_cursor_t cursor; + if (unw_getcontext(&_context) == 0 && unw_init_local(&cursor, &_context) == 0) { + butil::ignore_result(TraceCore(cursor)); } - return 0; + return true; } void TaskTracer::set_status(TaskStatus s, TaskMeta* m) { @@ -88,66 +201,39 @@ bool TaskTracer::set_end_status_unsafe(TaskMeta* m) { } std::string TaskTracer::Trace(bthread_t tid) { - Result result = TraceImpl(tid); - // return result.error ? result.err_msg : ToString(result); - if (result.error) { - return result.err_msg; - } - - if (result.frame_count == 0) { - return "No frame"; - } - - if (!result.fast_unwind) { - butil::debug::StackTrace stack_trace((void**)&result.ips, result.frame_count); - return stack_trace.ToString(); - } - - std::string trace_str; - trace_str.reserve(1024); - for (size_t i = 0; i < result.frame_count; ++i) { - butil::string_appendf(&trace_str, "#%zu 0x%016lx ", i, result.ips[i]); - if (strlen(result.mangled[i]) == 0) { - trace_str.append(""); - } else { - trace_str.append(butil::demangle(result.mangled[i])); - } - if (i + 1 < result.frame_count) { - trace_str.push_back('\n'); - } - } - return trace_str; + return TraceImpl(tid).OutputToString(); } void TaskTracer::Trace(std::ostream& os, bthread_t tid) { - Result result = TraceImpl(tid); - if (result.error) { - os << result.err_msg; - return; - } - - if (result.frame_count == 0) { - os << "No frame"; - return; - } + TraceImpl(tid).OutputToStream(os); +} - if (!result.fast_unwind) { - butil::debug::StackTrace stack_trace((void**)&result.ips, result.frame_count); - stack_trace.OutputToStream(&os); - return; +void TaskTracer::WaitForTracing(TaskMeta* m) { + BAIDU_SCOPED_LOCK(_mutex); + while (m->traced) { + _cond.Wait(); } +} - for (size_t i = 0; i < result.frame_count; ++i) { - os << "# " << i << " 0x" << std::hex << result.ips[i] << std::dec << " "; - if (strlen(result.mangled[i]) == 0) { - os << ""; +TaskStatus TaskTracer::WaitForJumping(TaskMeta* m) { + // Reasons for not using locks here: + // 1. It is necessary to lock before jump_stack, unlock after jump_stack, + // which involves two different bthread and is prone to errors. + // 2. jump_stack is fast. + int i = 0; + do { + // The bthread is jumping now, spin until it finishes. + if (i++ < 30) { + cpu_relax(); } else { - os << butil::demangle(result.mangled[i]); + sched_yield(); } - if (i + 1 < result.frame_count) { - os << '\n'; + + BAIDU_SCOPED_LOCK(m->version_lock); + if (TASK_STATUS_JUMPING != m->status) { + return m->status; } - } + } while (true); } TaskTracer::Result TaskTracer::TraceImpl(bthread_t tid) { @@ -219,45 +305,28 @@ TaskTracer::Result TaskTracer::TraceImpl(bthread_t tid) { return result; } -void TaskTracer::SignalSafeUsleep(unsigned int microseconds) { - ErrnoGuard guard; - struct timespec sleep_time{}; - sleep_time.tv_sec = microseconds / 1000000; - sleep_time.tv_nsec = (microseconds % 1000000) * 1000; - // On Linux, sleep() is implemented via nanosleep(2). - // sleep() is async-signal-safety, so nanosleep() is considered as async-signal-safety on Linux. - // abseil-cpp and folly also use nanosleep() in signal handler. For details, see: - // 1. abseil-cpp: https://github.com/abseil/abseil-cpp/blob/27a0c7308f04e4560fabe5a7beca837e8f3f2c5b/absl/debugging/failure_signal_handler.cc#L314 - // 2. folly: https://github.com/facebook/folly/blob/479de0144d3acb6aa4b3483affa23cf4f49f07ee/folly/debugging/symbolizer/SignalHandler.cpp#L446 - while (nanosleep(&sleep_time, &sleep_time) == -1 && EINTR == errno); -} +unw_cursor_t TaskTracer::MakeCursor(bthread_fcontext_t fcontext) { + unw_cursor_t cursor; + unw_init_local(&cursor, &_context); + auto regs = reinterpret_cast(fcontext); -void TaskTracer::WaitForTracing(TaskMeta* m) { - BAIDU_SCOPED_LOCK(_mutex); - while (m->traced) { - _cond.Wait(); + // Only need RBP, RIP, RSP on x86_64. + // The base pointer (RBP). + if (unw_set_reg(&cursor, UNW_X86_64_RBP, regs[6]) != 0) { + LOG(ERROR) << "Fail to set RBP"; } -} - -TaskStatus TaskTracer::WaitForJumping(TaskMeta* m) { - // Reasons for not using locks here: - // 1. It is necessary to lock before jump_stack, unlock after jump_stack, - // which involves two different bthread and is prone to errors. - // 2. jump_stack is fast. - int i = 0; - do { - // The bthread is jumping now, spin until it finishes. - if (i++ < 30) { - cpu_relax(); - } else { - sched_yield(); - } + // The instruction pointer (RIP). + if (unw_set_reg(&cursor, UNW_REG_IP, regs[7]) != 0) { + LOG(ERROR) << "Fail to set RIP"; + } +#if UNW_VERSION_MAJOR >= 1 && UNW_VERSION_MINOR >= 7 + // The stack pointer (RSP). + if (unw_set_reg(&cursor, UNW_REG_SP, regs[8]) != 0) { + LOG(ERROR) << "Fail to set RSP"; + } +#endif - BAIDU_SCOPED_LOCK(m->version_lock); - if (TASK_STATUS_JUMPING != m->status) { - return m->status; - } - } while (true); + return cursor; } TaskTracer::Result TaskTracer::ContextTrace(bthread_fcontext_t fcontext) { @@ -265,68 +334,112 @@ TaskTracer::Result TaskTracer::ContextTrace(bthread_fcontext_t fcontext) { return TraceCore(cursor); } -int TaskTracer::RegisterSignalHandler() { +bool TaskTracer::RegisterSignalHandler() { // Set up the signal handler. struct sigaction old_sa{}; struct sigaction sa{}; sa.sa_sigaction = SignalHandler; sa.sa_flags = SA_SIGINFO; sigfillset(&sa.sa_mask); - if (sigaction(SIGURG, &sa, &old_sa) == -1) { + if (sigaction(SIGURG, &sa, &old_sa) != 0) { PLOG(ERROR) << "Failed to sigaction"; - return -1; + return false; } if (NULL != old_sa.sa_handler || NULL != old_sa.sa_sigaction) { - LOG(ERROR) << "SIGURG is already registered"; - return -1; + LOG(ERROR) << "Signal handler of SIGURG is already registered"; + return false; } - return 0; + return true; } +// Caution: This function should be async-signal-safety. void TaskTracer::SignalHandler(int, siginfo_t* info, void* context) { ErrnoGuard guard; - TaskTracer* tracer = static_cast(info->si_value.sival_ptr); - if (NULL == tracer) { + butil::intrusive_ptr signal_sync( + static_cast(info->si_value.sival_ptr)); + if (NULL == signal_sync) { // The signal is not from Tracer, such as TaskControl, do nothing. return; } - tracer->SignalTraceHandler(static_cast(context)); -} -// Caution: This function is called in signal handler, so it should be async-signal-safety. -void TaskTracer::SignalTraceHandler(unw_context_t* context) { - // Something wrong, signal trace is not started, do nothing. - if (SIGNAL_TRACE_STATUS_START != - _signal_handler_flag.load(butil::memory_order_acquire)) { + signal_sync->context = static_cast(context); + // Notify SignalTrace that SignalTraceHandler has started. + // Binary semaphore do not fail, so no need to check return value. + // sem_post() is async-signal-safe. + sem_post(&signal_sync->sem); + + butil::Timer timer; + if (FLAGS_signal_trace_timeout_ms > 0) { + timer.start(); + } + int timeout = -1; + pollfd poll_fd = {signal_sync->pipe_fds[0], POLLIN, 0}; + // Wait for tracing to complete. + while (true) { + if (FLAGS_signal_trace_timeout_ms > 0) { + timer.stop(); + // At least 1ms timeout. + timeout = std::max( + (int64_t)FLAGS_signal_trace_timeout_ms - timer.m_elapsed(), (int64_t)1); + } + // poll() is async-signal-safe. + // Similar to self-pipe trick: https://man7.org/tlpi/code/online/dist/altio/self_pipe.c.html + int rc = poll(&poll_fd, 1, timeout); + if (-1 == rc && EINTR == errno) { + continue; + } + // No need to read the pipe or handle errors, just return. return; } +} - butil::Timer timer(butil::Timer::STARTED); - BRPC_SCOPE_EXIT { - timer.stop(); - _signal_handler_time << timer.n_elapsed(); - }; +// Caution: This fnction should be async-signal-safety. +bool TaskTracer::WaitForSignalHandler(butil::intrusive_ptr signal_sync, + const timespec* abs_timeout, Result& result) { + // It is safe to sem_timedwait() here and sem_post() in SignalHandler. + while (sem_timedwait(&signal_sync->sem, abs_timeout) != 0) { + if (EINTR == errno) { + continue; + } + if (ETIMEDOUT == errno) { + result.SetError("Timeout exceed %dms", FLAGS_signal_trace_timeout_ms); + } else { + // During the process of signal handler, + // can not use berro() which is not async-signal-safe. + result.SetError("Fail to sem_timedwait, errno=%d", errno); + } + return false; + } + return true; +} - _signal_handler_context = context; - // Use memory_order_seq_cst to ensure the flag is set before loop. - _signal_handler_flag.store(SIGNAL_TRACE_STATUS_TRACING, butil::memory_order_seq_cst); - while (SIGNAL_TRACE_STATUS_TRACING == - _signal_handler_flag.load(butil::memory_order_seq_cst)) { - SignalSafeUsleep(50); // 50us - // Timeout to avoid deadlock of libunwind. - timer.stop(); - if (timer.m_elapsed() > FLAGS_signal_trace_timeout_ms) { +// Caution: This fnction should be async-signal-safety. +void TaskTracer::WakeupSignalHandler(butil::intrusive_ptr signal_sync, Result& result) { + int try_count = 0; + while (true) { + ssize_t nw = write(signal_sync->pipe_fds[1], "1", 1); + if (0 < nw) { + break; + } else if (0 == nw) { + continue; + } else { + if (EINTR == errno) { + continue; + } else if (EAGAIN == errno && try_count++ < 3) { + usleep(1000); // Wait 1ms, try again. + continue; + } + // During the process of signal handler, + // can not use berro() which is not async-signal-safe. + result.SetError( + "Fail to write pipe to notify signal handler, errno=%d", errno); break; } } } TaskTracer::Result TaskTracer::SignalTrace(pid_t tid) { - _signal_handler_context = NULL; - // Use memory_order_seq_cst to ensure the flag is set before sending signal. - _signal_handler_flag.store(SIGNAL_TRACE_STATUS_START, butil::memory_order_seq_cst); - // CAUTION: // The signal handler will wait for the backtrace to complete. // If the worker thread is interrupted when holding a resource(lock, etc), @@ -345,8 +458,7 @@ TaskTracer::Result TaskTracer::SignalTrace(pid_t tid) { // #0 __lll_lock_wait (futex=futex@entry=0x7f0d3d7f0990 <_rtld_global+2352>, private=0) at lowlevellock.c:52 // #1 0x00007f0d3a73c131 in __GI___pthread_mutex_lock (mutex=0x7f0d3d7f0990 <_rtld_global+2352>) at ../nptl/pthread_mutex_lock.c:115 // #2 0x00007f0d38eb0231 in __GI___dl_iterate_phdr (callback=callback@entry=0x7f0d38c456a0 <_ULx86_64_dwarf_callback>, data=data@entry=0x7f0d07defad0) at dl-iteratephdr.c:40 - // #3 0x00007f0d38c45d79 in _ULx86_64_dwarf_find_proc_info (as=0x7f0d38c4f340 , ip=ip@entry=139694791966897, pi=pi@entry=0x7f0d07df0498, need_unwind_info=need_unwind_info@entry=1, arg=0x7f0 - // d07df0340) at dwarf/Gfind_proc_info-lsb.c:759 + // #3 0x00007f0d38c45d79 in _ULx86_64_dwarf_find_proc_info (as=0x7f0d38c4f340 , ip=ip@entry=139694791966897, pi=pi@entry=0x7f0d07df0498, need_unwind_info=need_unwind_info@entry=1, arg=0x7f0d07df0340) at dwarf/Gfind_proc_info-lsb.c:759 // #4 0x00007f0d38c43260 in fetch_proc_info (c=c@entry=0x7f0d07df0340, ip=139694791966897) at dwarf/Gparser.c:461 // #5 0x00007f0d38c44e46 in find_reg_state (sr=0x7f0d07defd10, c=0x7f0d07df0340) at dwarf/Gparser.c:925 // #6 _ULx86_64_dwarf_step (c=c@entry=0x7f0d07df0340) at dwarf/Gparser.c:972 @@ -374,8 +486,17 @@ TaskTracer::Result TaskTracer::SignalTrace(pid_t tid) { // backtracks with dl_iterate_phdr. We introduce a timeout mechanism in signal // handler to avoid deadlock. + // Each signal trace has an independent SignalSync to + // prevent the previous SignalHandler from affecting the new SignalTrace. + butil::intrusive_ptr signal_sync(new SignalSync()); + if (!signal_sync->Init()) { + return Result::MakeErrorResult("Fail to init SignalSync"); + } + // Add reference for SignalHandler. + signal_sync->AddRefManually(); + union sigval value{}; - value.sival_ptr = this; + value.sival_ptr = signal_sync.get(); size_t sigqueue_try = 0; while (sigqueue(tid, SIGURG, value) != 0) { if (errno != EAGAIN || sigqueue_try++ >= 3) { @@ -383,69 +504,39 @@ TaskTracer::Result TaskTracer::SignalTrace(pid_t tid) { } } - butil::Timer timer(butil::Timer::STARTED); - // Use memory_order_seq_cst to ensure the signal is sent and the flag is set before checking. - for (int i = 0; - SIGNAL_TRACE_STATUS_START == _signal_handler_flag.load(butil::memory_order_seq_cst); - ++i) { - if (i < 30) { - sched_yield(); - } else { - SignalSafeUsleep(5); // 5us - } + // Caution: Start here, need to ensure async-signal-safety. + Result result; + // Wakeup the signal handler at the end. + BRPC_SCOPE_EXIT { + WakeupSignalHandler(signal_sync, result); + }; - // Timeout to avoid dead loop if handler of SIGURG is covered. - timer.stop(); - if (timer.m_elapsed() > FLAGS_signal_trace_timeout_ms) { - return Result::MakeErrorResult( - "Timeout exceed %dms", FLAGS_signal_trace_timeout_ms); - } + timespec abs_timeout{}; + timespec* abs_timeout_ptr = NULL; + if (FLAGS_signal_trace_timeout_ms > 0) { + abs_timeout = butil::milliseconds_from_now(FLAGS_signal_trace_timeout_ms); + abs_timeout_ptr = &abs_timeout; } - - unw_cursor_t cursor; - int rc = unw_init_local(&cursor, _signal_handler_context); - Result result; - if (0 == rc) { - result = TraceCore(cursor); + // Wait for the signal handler to start. + if (!WaitForSignalHandler(signal_sync, abs_timeout_ptr, result)) { + return result; } - // Use memory_order_seq_cst to ensure the flag is set after tracing. - _signal_handler_flag.store(SIGNAL_TRACE_STATUS_UNKNOWN, butil::memory_order_seq_cst); - - return 0 == rc ? result : Result::MakeErrorResult("Failed to init local, rc=%d", rc); -} - -unw_cursor_t TaskTracer::MakeCursor(bthread_fcontext_t fcontext) { - unw_cursor_t cursor; - unw_init_local(&cursor, &_context); - auto regs = reinterpret_cast(fcontext); - - // Only need RBP, RIP, RSP on x86_64. - // The base pointer (RBP). - if (unw_set_reg(&cursor, UNW_X86_64_RBP, regs[6]) != 0) { - LOG(ERROR) << "Fail to set RBP"; + if (NULL == signal_sync->context) { + result.SetError("context is NULL"); + return result; } - // The instruction pointer (RIP). - if (unw_set_reg(&cursor, UNW_REG_IP, regs[7]) != 0) { - LOG(ERROR) << "Fail to set RIP"; - } -#if UNW_VERSION_MAJOR >= 1 && UNW_VERSION_MINOR >= 7 - // The stack pointer (RSP). - if (unw_set_reg(&cursor, UNW_REG_SP, regs[8]) != 0) { - LOG(ERROR) << "Fail to set RSP"; + unw_cursor_t cursor; + int rc = unw_init_local(&cursor, signal_sync->context); + if (0 != rc) { + result.SetError("Failed to init local, rc=%d", rc); + return result; } -#endif - return cursor; + return TraceCore(cursor); } TaskTracer::Result TaskTracer::TraceCore(unw_cursor_t& cursor) { - butil::Timer timer(butil::Timer::STARTED); - BRPC_SCOPE_EXIT { - timer.stop(); - _unwind_time << timer.n_elapsed(); - }; - Result result{}; result.fast_unwind = FLAGS_enable_fast_unwind; for (result.frame_count = 0; result.frame_count < arraysize(result.ips); ++result.frame_count) { @@ -453,7 +544,7 @@ TaskTracer::Result TaskTracer::TraceCore(unw_cursor_t& cursor) { if (0 == rc) { break; } else if (rc < 0) { - return Result::MakeErrorResult("unw_step rc=%d", rc); + return Result::MakeErrorResult("Fail to unw_step, rc=%d", rc); } unw_word_t ip = 0; @@ -461,21 +552,21 @@ TaskTracer::Result TaskTracer::TraceCore(unw_cursor_t& cursor) { rc = unw_get_reg(&cursor, UNW_REG_IP, &ip); result.ips[result.frame_count] = ip; - if (!result.fast_unwind) { + if (result.fast_unwind) { continue; } if (0 != rc) { - butil::strings::SafeSPrintf(result.mangled[result.frame_count], "\0"); + result.mangled[result.frame_count][0] = '\0'; continue; } // Slow path. rc = unw_get_proc_name(&cursor, result.mangled[result.frame_count], sizeof(result.mangled[result.frame_count]), NULL); - // UNW_ENOMEM is ok. + // UNW_ENOMEM is OK. if (0 != rc && UNW_ENOMEM != rc) { - butil::strings::SafeSPrintf(result.mangled[result.frame_count], "\0"); + result.mangled[result.frame_count][0] = '\0'; } } diff --git a/src/bthread/task_tracer.h b/src/bthread/task_tracer.h index 2f332f5a31..8c84f7b9a7 100644 --- a/src/bthread/task_tracer.h +++ b/src/bthread/task_tracer.h @@ -15,16 +15,20 @@ // specific language governing permissions and limitations // under the License. -#ifndef BRPC_BTHREAD_TRACER_H -#define BRPC_BTHREAD_TRACER_H +#ifndef BTHREAD_TASK_TRACER_H +#define BTHREAD_TASK_TRACER_H #ifdef BRPC_BTHREAD_TRACER #include +#include #include +#include #include #include "butil/strings/safe_sprintf.h" #include "butil/synchronization/condition_variable.h" +#include "butil/shared_object.h" +#include "butil/fd_utility.h" #include "bthread/task_meta.h" #include "bthread/mutex.h" @@ -34,15 +38,12 @@ namespace bthread { class TaskTracer { public: // Returns 0 on success, -1 otherwise. - int Init(); + bool Init(); // Set the status to `s'. void set_status(TaskStatus s, TaskMeta* meta); static void set_running_status(pid_t worker_tid, TaskMeta* meta); static bool set_end_status_unsafe(TaskMeta* m); - // Async signal safe usleep. - static void SignalSafeUsleep(unsigned int microseconds); - // Trace the bthread of `tid'. std::string Trace(bthread_t tid); void Trace(std::ostream& os, bthread_t tid); @@ -60,47 +61,66 @@ class TaskTracer { int _errno; }; - enum SignalTraceStatus { - SIGNAL_TRACE_STATUS_UNKNOWN = 0, - SIGNAL_TRACE_STATUS_START, - SIGNAL_TRACE_STATUS_TRACING, - }; - struct Result { template static Result MakeErrorResult(const char* fmt, Args... args) { Result result{}; - result.error = true; - butil::strings::SafeSPrintf(result.err_msg, fmt, args...); + result.SetError(fmt, std::forward(args)...); return result; } + template + void SetError(const char* fmt, Args... args) { + err_count = std::max(err_count + 1, MAX_ERROR_NUM); + butil::strings::SafeSPrintf(err_msg[err_count - 1], fmt, args...); + } + + std::string OutputToString(); + void OutputToStream(std::ostream& os); + + bool OK() const { return err_count == 0; } + static const size_t MAX_TRACE_NUM = 64; + static const size_t MAX_ERROR_NUM = 2; + unw_word_t ips[MAX_TRACE_NUM]; + char mangled[MAX_TRACE_NUM][256]{}; size_t frame_count{0}; - bool error{false}; - union { - char mangled[MAX_TRACE_NUM][256]{}; - char err_msg[256]; - }; + char err_msg[MAX_ERROR_NUM][64]{}; + size_t err_count{0}; bool fast_unwind{false}; }; - Result TraceImpl(bthread_t tid); + // For signal trace. + struct SignalSync : public butil::SharedObject { + ~SignalSync() override; + bool Init(); + + unw_context_t* context{NULL}; + sem_t sem{}; + int pipe_fds[2]{}; + + private: + bool _pipe_init{false}; + bool _sem_init{false}; + }; static TaskStatus WaitForJumping(TaskMeta* m); + Result TraceImpl(bthread_t tid); + unw_cursor_t MakeCursor(bthread_fcontext_t fcontext); Result ContextTrace(bthread_fcontext_t fcontext); - // Register signal handler for signal trace. - static int RegisterSignalHandler(); + static bool RegisterSignalHandler(); static void SignalHandler(int sig, siginfo_t* info, void* context); - void SignalTraceHandler(unw_context_t* context); + static bool WaitForSignalHandler(butil::intrusive_ptr signal_sync, + const timespec* abs_timeout, Result& result); + static void WakeupSignalHandler( + butil::intrusive_ptr signal_sync, Result& result); Result SignalTrace(pid_t worker_tid); - unw_cursor_t MakeCursor(bthread_fcontext_t fcontext); - Result TraceCore(unw_cursor_t& cursor); + static Result TraceCore(unw_cursor_t& cursor); // Make sure only one bthread is traced at a time. bthread::Mutex _trace_request_mutex; @@ -112,17 +132,8 @@ class TaskTracer { // For context trace. unw_context_t _context{}; - // For signal trace. - unw_context_t* _signal_handler_context{NULL}; - butil::atomic _signal_handler_flag{SIGNAL_TRACE_STATUS_UNKNOWN}; - - // Protect `_worker_tids'. - butil::Mutex _worker_mutex; - std::vector _worker_tids; bvar::LatencyRecorder _trace_time{"bthread_trace_time"}; - bvar::LatencyRecorder _unwind_time{"bthread_unwind_time"}; - bvar::LatencyRecorder _signal_handler_time{"bthread_signal_handler_time"}; }; } // namespace bthread diff --git a/src/butil/memory/scope_guard.h b/src/butil/memory/scope_guard.h index 837acbbca1..7d72a560d2 100644 --- a/src/butil/memory/scope_guard.h +++ b/src/butil/memory/scope_guard.h @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#ifndef BRPC_SCOPED_GUARD_H -#define BRPC_SCOPED_GUARD_H +#ifndef BUTIL_SCOPED_GUARD_H +#define BUTIL_SCOPED_GUARD_H #include "butil/type_traits.h" #include "butil/macros.h" @@ -104,4 +104,4 @@ operator+(ScopeExitHelper, Callback&& callback) { auto BRPC_ANONYMOUS_VARIABLE(SCOPE_EXIT) = \ ::butil::internal::ScopeExitHelper() + [&]() noexcept -#endif // BRPC_SCOPED_GUARD_H +#endif // BUTIL_SCOPED_GUARD_H diff --git a/src/butil/reloadable_flags.h b/src/butil/reloadable_flags.h new file mode 100644 index 0000000000..9bb62d82f1 --- /dev/null +++ b/src/butil/reloadable_flags.h @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#ifndef BUTIL_RELOADABLE_FLAGS_H +#define BUTIL_RELOADABLE_FLAGS_H + +#include +#include // write, _exit +#include +#include "butil/macros.h" +#include "butil/type_traits.h" + +// Register an always-true valiator to a gflag so that the gflag is treated as +// reloadable by brpc. If a validator exists, abort the program. +// You should call this macro within global scope. for example: +// +// DEFINE_int32(foo, 0, "blah blah"); +// BRPC_VALIDATE_GFLAG(foo, brpc::PassValidate); +// +// This macro does not work for string-flags because they're thread-unsafe to +// modify directly. To emphasize this, you have to write the validator by +// yourself and use GFLAGS_NS::GetCommandLineOption() to acess the flag. +#define BRPC_VALIDATE_GFLAG(flag, validate_fn) \ + namespace butil_flags {} \ + const int register_FLAGS_ ## flag ## _dummy \ + __attribute__((__unused__)) = \ + ::butil::RegisterFlagValidatorOrDieImpl< \ + decltype(FLAGS_##flag)>(&FLAGS_##flag, (validate_fn)) + + +namespace butil { + +template +bool PassValidate(const char*, T) { + return true; +} + +template +bool PositiveInteger(const char*, T v) { + return v > 0; +} + +template +bool RegisterFlagValidatorOrDieImpl( + const T* flag, bool (*validate_fn)(const char*, T val)) { + static_assert(!butil::is_same::value, + "Not support string flags"); + if (GFLAGS_NS::RegisterFlagValidator(flag, validate_fn)) { + return true; + } + // Error printed by gflags does not have newline. Add one to it. + char newline = '\n'; + butil::ignore_result(write(2, &newline, 1)); + _exit(1); +} + +} // namespace butil + + +#endif // BUTIL_RELOADABLE_FLAGS_H diff --git a/src/butil/shared_object.h b/src/butil/shared_object.h new file mode 100644 index 0000000000..abcfd46c4b --- /dev/null +++ b/src/butil/shared_object.h @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#ifndef BUTIL_SHARED_OBJECT_H +#define BUTIL_SHARED_OBJECT_H + +#include "butil/intrusive_ptr.hpp" // butil::intrusive_ptr +#include "butil/atomicops.h" + + +namespace butil { + +// Inherit this class to be intrusively shared. Comparing to shared_ptr, +// intrusive_ptr saves one malloc (for shared_count) and gets better cache +// locality when the ref/deref are frequent, in the cost of lack of weak_ptr +// and worse interface. +class SharedObject { +friend void intrusive_ptr_add_ref(SharedObject*); +friend void intrusive_ptr_release(SharedObject*); + +public: + SharedObject() : _nref(0) { } + int ref_count() const { return _nref.load(butil::memory_order_relaxed); } + + // Add ref and returns the ref_count seen before added. + // The effect is basically same as butil::intrusive_ptr(obj).detach() + // except that the latter one does not return the seen ref_count which is + // useful in some scenarios. + int AddRefManually() + { return _nref.fetch_add(1, butil::memory_order_relaxed); } + + // Remove one ref, if the ref_count hit zero, delete this object. + // Same as butil::intrusive_ptr(obj, false).reset(NULL) + void RemoveRefManually() { + if (_nref.fetch_sub(1, butil::memory_order_release) == 1) { + butil::atomic_thread_fence(butil::memory_order_acquire); + delete this; + } + } + +protected: + virtual ~SharedObject() { } +private: + butil::atomic _nref; +}; + +inline void intrusive_ptr_add_ref(SharedObject* obj) { + obj->AddRefManually(); +} + +inline void intrusive_ptr_release(SharedObject* obj) { + obj->RemoveRefManually(); +} + +} // namespace butil + + +#endif // BUTIL_SHARED_OBJECT_H diff --git a/test/bthread_unittest.cpp b/test/bthread_unittest.cpp index 0283875eb6..781b0c5ddd 100644 --- a/test/bthread_unittest.cpp +++ b/test/bthread_unittest.cpp @@ -34,6 +34,7 @@ int main(int argc, char* argv[]) { namespace bthread { extern __thread bthread::LocalStorage tls_bls; +DECLARE_bool(enable_fast_unwind); #ifdef BRPC_BTHREAD_TRACER extern std::string stack_trace(bthread_t tid); #endif // BRPC_BTHREAD_TRACER @@ -622,8 +623,14 @@ TEST_F(BthreadTest, trace) { stop = false; bthread_t th; ASSERT_EQ(0, bthread_start_urgent(&th, NULL, spin_and_log, (void*)1)); - usleep(100 * 1000); + usleep(10 * 1000); + bthread::FLAGS_enable_fast_unwind = false; std::string st = bthread::stack_trace(th); + LOG(INFO) << "fast_unwind spin_and_log stack trace:\n" << st; + ASSERT_NE(std::string::npos, st.find("spin_and_log")); + + bthread::FLAGS_enable_fast_unwind = true; + st = bthread::stack_trace(th); LOG(INFO) << "spin_and_log stack trace:\n" << st; ASSERT_NE(std::string::npos, st.find("spin_and_log")); stop = true; @@ -632,6 +639,12 @@ TEST_F(BthreadTest, trace) { stop = false; ASSERT_EQ(0, bthread_start_urgent(&th, NULL, repeated_sleep, (void*)1)); usleep(100 * 1000); + bthread::FLAGS_enable_fast_unwind = false; + st = bthread::stack_trace(th); + LOG(INFO) << "repeated_sleep stack trace:\n" << st; + ASSERT_NE(std::string::npos, st.find("repeated_sleep")); + + bthread::FLAGS_enable_fast_unwind = true; st = bthread::stack_trace(th); LOG(INFO) << "repeated_sleep stack trace:\n" << st; ASSERT_NE(std::string::npos, st.find("repeated_sleep"));