DataDog · nsavoire · Apr 2, 2024 · Jan 11, 2024 · Jan 26, 2024 · Jan 26, 2024
@@ -61,3 +61,4 @@ CheckOptions:
     cppcoreguidelines-special-member-functions.AllowMissingMoveFunctionsWhenCopyIsDeleted: true
     misc-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic: true
     readability-magic-numbers.IgnorePowersOf2IntegerValues: true
+    readability-simplify-boolean-expr.IgnoreMacros: true
@@ -207,21 +207,20 @@ if(USE_LOADER)
 endif()
 
 set(DD_PROFILING_SOURCES
-    # cmake-format: sortable
     src/daemonize.cc
     src/ddprof_cmdline.cc
     src/ddres_list.cc
     src/ipc.cc
     src/lib/address_bitset.cc
     src/lib/allocation_tracker.cc
-    src/lib/dd_profiling.cc
     src/lib/elfutils.cc
     src/lib/lib_embedded_data.c
     src/lib/pthread_fixes.cc
     src/lib/savecontext.cc
     src/lib/saveregisters.cc
     src/lib/symbol_overrides.cc
     src/logger.cc
+    src/logger_setup.cc
     src/perf.cc
     src/perf_clock.cc
     src/perf_ringbuffer.cc
@@ -233,7 +232,10 @@ set(DD_PROFILING_SOURCES
     src/sys_utils.cc
     src/tracepoint_config.cc
     src/tsc_clock.cc
-    src/user_override.cc)
+    src/user_override.cc
+    # Intentionally put dd_profiling.cc last to avoid leak sanitizer failure complaining that memory
+    # in LoggerContext::name is not freed
+    src/lib/dd_profiling.cc)
 
 if(BUILD_UNIVERSAL_DDPROF)
   # Compiling on different libc, we need to ensure some symbols are available everywhere
@@ -246,6 +248,7 @@ target_include_directories(dd_profiling-embedded PUBLIC ${CMAKE_SOURCE_DIR}/incl
                                                         ${CMAKE_SOURCE_DIR}/include)
 set_target_properties(dd_profiling-embedded
                       PROPERTIES PUBLIC_HEADER "${CMAKE_SOURCE_DIR}/include/lib/dd_profiling.h")
+target_compile_definitions(dd_profiling-embedded PRIVATE DDPROF_PROFILING_LIBRARY)
 
 # Link libstdc++/libgcc statically and export only profiler API
 target_static_libcxx(dd_profiling-embedded)
@@ -261,7 +264,6 @@ if(BUILD_UNIVERSAL_DDPROF)
     target_link_options(
       dd_profiling-embedded PRIVATE
       "-Wl,-f,libpthread.so.0;-Wl,-f,libm.so.6;-Wl,-f,libdl.so.2;-Wl,-f,librt.so.1")
-    target_compile_definitions(dd_profiling-embedded PRIVATE DDPROF_MISSING_FSTAT)
   endif()
 endif()
 
@@ -356,7 +358,8 @@ add_dependencies(ddprof_exe_object generate_ddprof_object)
 
 add_library(dd_profiling-static STATIC ${DD_PROFILING_SOURCES})
 set_target_properties(dd_profiling-static PROPERTIES OUTPUT_NAME dd_profiling)
-target_compile_definitions(dd_profiling-static PRIVATE DDPROF_EMBEDDED_EXE_DATA)
+target_compile_definitions(dd_profiling-static PRIVATE DDPROF_EMBEDDED_EXE_DATA
+                                                       DDPROF_PROFILING_LIBRARY)
 target_include_directories(dd_profiling-static PUBLIC ${CMAKE_SOURCE_DIR}/include/lib
                                                       ${CMAKE_SOURCE_DIR}/include)
 set_target_properties(dd_profiling-static
@@ -371,14 +374,16 @@ if(USE_LOADER)
                                          src/lib/loader.c)
   target_link_libraries(dd_profiling-shared PRIVATE libddprofiling_embedded_object
                                                     ddprof_exe_object)
-  target_compile_definitions(dd_profiling-shared PRIVATE DDPROF_EMBEDDED_LIB_DATA
-                                                         DDPROF_EMBEDDED_EXE_DATA)
+  target_compile_definitions(
+    dd_profiling-shared PRIVATE DDPROF_EMBEDDED_LIB_DATA DDPROF_EMBEDDED_EXE_DATA
+                                DDPROF_PROFILING_LIBRARY)
 else()
   # Without loader, libdd_profiling.so is basically the same as libdd_profiling-embedded.so plus an
   # embedded ddprof executable.
   add_library(dd_profiling-shared SHARED ${DD_PROFILING_SOURCES} src/lib/lib_embedded_data.c)
   target_link_libraries(dd_profiling-shared PRIVATE ddprof_exe_object)
-  target_compile_definitions(dd_profiling-shared PRIVATE DDPROF_EMBEDDED_EXE_DATA)
+  target_compile_definitions(dd_profiling-shared PRIVATE DDPROF_EMBEDDED_EXE_DATA
+                                                         DDPROF_PROFILING_LIBRARY)
 
   # Fix for link error in sanitizeddebug build mode with gcc:
   # ~~~

@@ -142,7 +142,8 @@ inline int ddres_sev_to_log_level(int sev) {
 // workaround once it is released. \fixme{nsavoire}
 #define DDPROF_CHECK_FATAL_IMPL(condition, condition_text, text, ...)          \
   do {                                                                         \
-    if (unlikely(!(condition))) {                                              \
+    if (unlikely(                                                              \
+            !(condition))) { /* NOLINT(readability-simplify-boolean-expr) */   \
       LG_IF_LVL_OK(LL_CRITICAL, "Check failed: `%s`. " text, condition_text,   \
                    ##__VA_ARGS__);                                             \
       std::abort();                                                            \
@@ -152,7 +153,8 @@ inline int ddres_sev_to_log_level(int sev) {
 #ifndef NDEBUG
 #  define DDPROF_DCHECK_FATAL_IMPL(condition, condition_text, text, ...)       \
     do {                                                                       \
-      if (unlikely(!(condition))) {                                            \
+      if (unlikely(                                                            \
+              !(condition))) { /* NOLINT(readability-simplify-boolean-expr) */ \
         LG_IF_LVL_OK(LL_CRITICAL, "Check failed: `%s`. " text, condition_text, \
                      ##__VA_ARGS__);                                           \
         std::abort();                                                          \
@@ -164,10 +166,17 @@ inline int ddres_sev_to_log_level(int sev) {
     } while (0)
 #endif
 
+#ifndef DDPROF_PROFILING_LIBRARY
 // Fatal assertion check that terminates the program with a fatal error if
 // `condition` is not true.
-#define DDPROF_CHECK_FATAL(condition, ...)                                     \
-  DDPROF_CHECK_FATAL_IMPL(condition, #condition, __VA_ARGS__)
+#  define DDPROF_CHECK_FATAL(condition, ...)                                   \
+    DDPROF_CHECK_FATAL_IMPL(condition, #condition, __VA_ARGS__)
+#else
+// DDPROF_CHECK_FATAL must not be used inside profiling library
+#  define DDPROF_CHECK_FATAL(condition, ...)                                   \
+    static_assert(                                                             \
+        false, "DDPROF_CHECK_FATAL must not be used inside profiling library")
+#endif
 
 // `DDPROF_DCHECK_FATAL` behaves like `DDPROF_CHECK_FATAL` in debug mode but
 // does nothing otherwise (if NDEBUG is defined)

@@ -5,12 +5,13 @@
 
 #pragma once
 
-#include <chrono>
-#include <stdarg.h>
-
 #include "unlikely.hpp"
 #include "version.hpp"
 
+#include <chrono>
+#include <functional>
+#include <stdarg.h>
+
 namespace ddprof {
 
 enum LOG_OPTS {
@@ -99,6 +100,13 @@ void LOG_setfacility(int fac);
 void LOG_setratelimit(uint64_t max_log_per_interval,
                       std::chrono::nanoseconds interval);
 
+bool LOG_is_logging_enabled_for_level(int level);
+
+using LogsAllowedCallback = std::function<bool()>;
+
+// Allow to inject a function used by logger to check if logs are allowed
+void LOG_set_logs_allowed_function(LogsAllowedCallback logs_allowed_function);
+
 /******************************* Logging Macros *******************************/
 #define ABS(__x)                                                               \
   ({                                                                           \
@@ -109,7 +117,7 @@ void LOG_setratelimit(uint64_t max_log_per_interval,
 // Avoid calling arguments (which can have CPU costs unless level is OK)
 #define LG_IF_LVL_OK(level, ...)                                               \
   do {                                                                         \
-    if (unlikely(level <= LOG_getlevel())) {                                   \
+    if (unlikely(LOG_is_logging_enabled_for_level(level))) {                   \
       olprintfln(ABS(level), -1, MYNAME, __VA_ARGS__);                         \
     }                                                                          \
   } while (false)

@@ -50,9 +50,10 @@ TrackerThreadLocalState *AllocationTracker::init_tl_state() {
   tl_state->tid = ddprof::gettid();
   tl_state->stack_bounds = retrieve_stack_bounds();
 
-  if (int res = pthread_setspecific(_tl_state_key, tl_state.get()); res != 0) {
+  if (int const res = pthread_setspecific(_tl_state_key, tl_state.get());
+      res != 0) {
     // should return 0
-    LOG_ONCE("Error: Unable to store tl_state. error %d \n", res);
+    LG_ERR("Unable to store tl_state. Error %d: %s\n", res, strerror(res));
     tl_state.reset();
   }
 
@@ -176,8 +177,7 @@ void AllocationTracker::allocation_tracking_free() {
   }
   TrackerThreadLocalState *tl_state = get_tl_state();
   if (unlikely(!tl_state)) {
-    const char *func_name = __FUNCTION__;
-    LOG_ONCE("Error: Unable to find tl_state during %s\n", func_name);
+    LG_ERR("Unable to get tl_state during allocation_tracking_free\n");
     instance->free();
     return;
   }
@@ -247,9 +247,8 @@ void AllocationTracker::track_allocation(uintptr_t addr, size_t /*size*/,
           // still set this as we are pushing the allocation to ddprof
           _allocated_address_set.add(addr);
         } else {
-          LOG_ONCE(
-              "Error: %s",
-              "Stop allocation profiling. Unable to clear live allocation \n");
+          LG_ERR("Stopping allocation profiling. Unable to clear live "
+                 "allocation\n");
           free();
         }
       }
@@ -304,6 +303,9 @@ DDRes AllocationTracker::push_lost_sample(MPSCRingBufferWriter &writer,
   event->header.type = PERF_RECORD_LOST;
   auto now = PerfClock::now();
   event->sample_id.time = now.time_since_epoch().count();
+
+  DDPROF_DCHECK_FATAL(_state.pid != 0 && tl_state.tid != 0,
+                      "pid or tid is not set");
   event->sample_id.pid = _state.pid;
   event->sample_id.tid = tl_state.tid;
 
@@ -337,6 +339,9 @@ DDRes AllocationTracker::push_clear_live_allocation(
   event->hdr.type = PERF_CUSTOM_EVENT_CLEAR_LIVE_ALLOCATION;
   auto now = PerfClock::now();
   event->sample_id.time = now.time_since_epoch().count();
+
+  DDPROF_DCHECK_FATAL(_state.pid != 0 && tl_state.tid != 0,
+                      "pid or tid is not set");
   event->sample_id.pid = _state.pid;
   event->sample_id.tid = tl_state.tid;
 
@@ -383,6 +388,9 @@ DDRes AllocationTracker::push_dealloc_sample(
   event->hdr.type = PERF_CUSTOM_EVENT_DEALLOCATION;
   auto now = PerfClock::now();
   event->sample_id.time = now.time_since_epoch().count();
+
+  DDPROF_DCHECK_FATAL(_state.pid != 0 && tl_state.tid != 0,
+                      "pid or tid is not set");
   event->sample_id.pid = _state.pid;
   event->sample_id.tid = tl_state.tid;
 
@@ -463,6 +471,9 @@ DDRes AllocationTracker::push_alloc_sample(uintptr_t addr,
   auto now = PerfClock::now();
   event->sample_id.time = now.time_since_epoch().count();
   event->addr = addr;
+
+  DDPROF_DCHECK_FATAL(_state.pid != 0 && tl_state.tid != 0,
+                      "pid or tid is not set");
   event->sample_id.pid = _state.pid;
   event->sample_id.tid = tl_state.tid;
   event->period = allocated_size;
@@ -539,8 +550,8 @@ void AllocationTracker::notify_thread_start() {
   if (unlikely(!tl_state)) {
     tl_state = init_tl_state();
     if (!tl_state) {
-      LOG_ONCE("Error: Unable to start allocation profiling on thread %d",
-               ddprof::gettid());
+      LG_ERR("Unable to start allocation profiling on thread %d",
+             ddprof::gettid());
       return;
     }
   }
@@ -554,8 +565,8 @@ void AllocationTracker::notify_fork() {
   if (unlikely(!tl_state)) {
     // The state should already exist if we forked.
     // This would mean that we were not able to create the state before forking
-    LOG_ONCE("Error: Unable to retrieve tl state after fork thread %d",
-             ddprof::gettid());
+    LG_ERR("Unable to retrieve tl state after fork thread %d",
+           ddprof::gettid());
     return;
   }
   tl_state->tid = ddprof::gettid();

@@ -109,6 +109,40 @@ void init_profiler_library_active() {
   }
 }
 
+std::string get_log_mode() {
+  auto *log_mode_env = g_state.getenv("DD_PROFILING_NATIVE_LIBRARY_LOG_MODE");
+
+  if (!log_mode_env) {
+    log_mode_env = g_state.getenv("DD_PROFILING_NATIVE_LOG_MODE");
+  }
+
+  return log_mode_env ? log_mode_env : "stderr";
+}
+
+std::string get_log_level() {
+  auto *log_level_env = g_state.getenv("DD_PROFILING_NATIVE_LIBRARY_LOG_LEVEL");
+
+  if (!log_level_env) {
+    log_level_env = g_state.getenv("DD_PROFILING_NATIVE_LOG_LEVEL");
+  }
+
+  return log_level_env ? log_level_env : "error";
+}
+
+void init_logger() {
+  auto log_level = get_log_level();
+  auto log_mode = get_log_mode();
+  setup_logger(log_mode.c_str(), log_level.c_str());
+
+  LOG_setname("ddprof-library");
+
+  // Disable logging when allocations are not allowed
+  LOG_set_logs_allowed_function([]() {
+    auto *tl_state = AllocationTracker::get_tl_state();
+    return !tl_state || tl_state->allocation_allowed;
+  });
+}
+
 void init_state() {
   if (g_state.initialized) {
     return;
@@ -119,6 +153,8 @@ void init_state() {
 
   auto *follow_execs_env = g_state.getenv(k_allocation_profiling_follow_execs);
   g_state.follow_execs = !(follow_execs_env && arg_no(follow_execs_env));
+
+  init_logger();
   g_state.initialized = true;
 }
 
@@ -363,15 +399,15 @@ int ddprof_start_profiling_internal() {
         // fails ?
         g_state.allocation_profiling_started = true;
       } else {
-        LOG_ONCE("Error: %s", "Failure to start allocation profiling\n");
+        LG_ERR("Failed to start allocation profiling\n");
       }
     }
   } catch (const DDException &e) { return -1; }
 
   if (g_state.allocation_profiling_started) {
     int const res = pthread_atfork(nullptr, nullptr, notify_fork);
     if (res) {
-      LOG_ONCE("Error:%s", "Unable to setup notify fork");
+      LG_ERR("Unable to setup notify fork. Error: %d: %s", res, strerror(res));
       assert(0);
     }
   }

@@ -15,10 +15,10 @@ extern const unsigned char _binary_libdd_profiling_embedded_so_start[];
 extern const unsigned char _binary_libdd_profiling_embedded_so_end[];
 #else
 // NOLINTNEXTLINE(cert-dcl51-cpp)
-static const unsigned char _binary_libdd_profiling_embedded_so_start[] = {};
+static const unsigned char *_binary_libdd_profiling_embedded_so_start = 0;
 // NOLINTNEXTLINE(cert-dcl51-cpp)
+static const unsigned char *_binary_libdd_profiling_embedded_so_end = 0;
 static const char *libdd_profiling_embedded_hash = "";
-static const unsigned char _binary_libdd_profiling_embedded_so_end[] = {};
 #endif
 
 #ifdef DDPROF_EMBEDDED_EXE_DATA
@@ -28,9 +28,8 @@ static const unsigned char _binary_libdd_profiling_embedded_so_end[] = {};
 extern const unsigned char _binary_ddprof_start[]; // NOLINT(cert-dcl51-cpp)
 extern const unsigned char _binary_ddprof_end[];   // NOLINT(cert-dcl51-cpp)
 #else
-static const unsigned char _binary_ddprof_start[] =
-    {};                                               // NOLINT(cert-dcl51-cpp)
-static const unsigned char _binary_ddprof_end[] = {}; // NOLINT(cert-dcl51-cpp)
+static const unsigned char *_binary_ddprof_start = 0; // NOLINT(cert-dcl51-cpp)
+static const unsigned char *_binary_ddprof_end = 0;   // NOLINT(cert-dcl51-cpp)
 static const char *ddprof_exe_hash = "";
 #endif
 

@@ -45,6 +45,7 @@ struct LoggerContext {
   int facility{LF_USER};
   std::string name;
   std::optional<IntervalRateLimiter> rate_limiter;
+  LogsAllowedCallback logs_allowed_function;
 };
 
 LoggerContext log_ctx{.fd = -1, .mode = LOG_STDERR, .level = LL_ERROR};
@@ -149,10 +150,6 @@ void vlprintfln(int lvl, int fac, const char *name, const char *format,
   ssize_t sz_h = -1;
   int rc = 0;
 
-  if (log_ctx.rate_limiter && !log_ctx.rate_limiter->check()) {
-    return;
-  }
-
   // Special value handling
   if (lvl == -1) {
     lvl = log_ctx.level;
@@ -251,4 +248,14 @@ void lprintfln(int lvl, int fac, const char *name, const char *fmt, ...) {
   va_end(args);
 }
 
+void LOG_set_logs_allowed_function(LogsAllowedCallback logs_allowed_function) {
+  log_ctx.logs_allowed_function = std::move(logs_allowed_function);
+}
+
+bool LOG_is_logging_enabled_for_level(int level) {
+  return (level <= log_ctx.level) &&
+      (!log_ctx.logs_allowed_function || log_ctx.logs_allowed_function()) &&
+      (!log_ctx.rate_limiter || log_ctx.rate_limiter->check());
+}
+
 } // namespace ddprof
@@ -78,7 +78,7 @@ check_logs() {
             fi
         done
     else
-        if [ -f "${log_file}" ]; then
+        if [ -s "${log_file}" ]; then
             echo "Unexpected samples for $cmd"
             cat "${log_file}"
             exit 1