diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/QueueTimerHelper.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/QueueTimerHelper.java index 5d5ab8c79e1..2f77a3fb4bf 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/QueueTimerHelper.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/QueueTimerHelper.java @@ -30,16 +30,22 @@ private static final class RateLimiterHolder { } public static void startQueuingTimer( - ContextStore taskContextStore, Class schedulerClass, T task) { + ContextStore taskContextStore, + Class schedulerClass, + Class queueClass, + int queueLength, + T task) { State state = taskContextStore.get(task); - startQueuingTimer(state, schedulerClass, task); + startQueuingTimer(state, schedulerClass, queueClass, queueLength, task); } - public static void startQueuingTimer(State state, Class schedulerClass, Object task) { + public static void startQueuingTimer( + State state, Class schedulerClass, Class queueClass, int queueLength, Object task) { if (Platform.isNativeImage()) { // explicitly not supported for Graal native image return; } + // TODO consider queue length based sampling here to reduce overhead // avoid calling this before JFR is initialised because it will lead to reading the wrong // TSC frequency before JFR has set it up properly if (task != null && state != null && InstrumentationBasedProfiling.isJFRReady()) { @@ -47,6 +53,8 @@ public static void startQueuingTimer(State state, Class schedulerClass, Objec (QueueTiming) AgentTracer.get().getProfilingContext().start(Timer.TimerType.QUEUEING); timing.setTask(task); timing.setScheduler(schedulerClass); + timing.setQueue(queueClass); + timing.setQueueLength(queueLength); state.setTiming(timing); } } diff --git a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/events/QueueTimeEvent.java b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/events/QueueTimeEvent.java index 67dbd7781a0..54b4f09b55f 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/events/QueueTimeEvent.java +++ b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/events/QueueTimeEvent.java @@ -33,6 +33,12 @@ public class QueueTimeEvent extends Event implements QueueTiming { @Label("Scheduler") private Class scheduler; + @Label("Queue") + private Class queueType; + + @Label("Queue Length on Entry") + private int queueLength; + public QueueTimeEvent() { this.origin = Thread.currentThread(); AgentSpan activeSpan = AgentTracer.activeSpan(); @@ -55,6 +61,16 @@ public void setScheduler(Class scheduler) { this.scheduler = scheduler; } + @Override + public void setQueue(Class queueType) { + this.queueType = queueType; + } + + @Override + public void setQueueLength(int queueLength) { + this.queueLength = queueLength; + } + @Override public void report() { commit(); diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index 21c7eafc206..2e959387796 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -451,14 +451,21 @@ boolean shouldRecordQueueTimeEvent(long startMillis) { return System.currentTimeMillis() - startMillis >= queueTimeThresholdMillis; } - void recordQueueTimeEvent(long startTicks, Object task, Class scheduler, Thread origin) { + void recordQueueTimeEvent( + long startTicks, + Object task, + Class scheduler, + Class queueType, + int queueLength, + Thread origin) { if (profiler != null) { // note: because this type traversal can update secondary_super_cache (see JDK-8180450) // we avoid doing this unless we are absolutely certain we will record the event Class taskType = TaskWrapper.getUnwrappedType(task); if (taskType != null) { long endTicks = profiler.getCurrentTicks(); - profiler.recordQueueTime(startTicks, endTicks, taskType, scheduler, origin); + profiler.recordQueueTime( + startTicks, endTicks, taskType, scheduler, queueType, queueLength, origin); } } } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java index fb25738ac7d..e76a3dd5561 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java @@ -13,6 +13,8 @@ public class QueueTimeTracker implements QueueTiming { // FIXME this can be eliminated by altering the instrumentation // since it is known when the item is polled from the queue private Class scheduler; + private Class queue; + private int queueLength; public QueueTimeTracker(DatadogProfiler profiler, long startTicks) { this.profiler = profiler; @@ -31,13 +33,23 @@ public void setScheduler(Class scheduler) { this.scheduler = scheduler; } + @Override + public void setQueue(Class queue) { + this.queue = queue; + } + + @Override + public void setQueueLength(int queueLength) { + this.queueLength = queueLength; + } + @Override public void report() { assert weakTask != null && scheduler != null; Object task = this.weakTask.get(); if (task != null) { // indirection reduces shallow size of the tracker instance - profiler.recordQueueTimeEvent(startTicks, task, scheduler, origin); + profiler.recordQueueTimeEvent(startTicks, task, scheduler, queue, queueLength, origin); } } diff --git a/dd-java-agent/instrumentation/grpc-1.5/src/main/java/datadog/trace/instrumentation/grpc/QueuedCommandInstrumentation.java b/dd-java-agent/instrumentation/grpc-1.5/src/main/java/datadog/trace/instrumentation/grpc/QueuedCommandInstrumentation.java index 0f0642e0612..506c3414b47 100644 --- a/dd-java-agent/instrumentation/grpc-1.5/src/main/java/datadog/trace/instrumentation/grpc/QueuedCommandInstrumentation.java +++ b/dd-java-agent/instrumentation/grpc-1.5/src/main/java/datadog/trace/instrumentation/grpc/QueuedCommandInstrumentation.java @@ -20,6 +20,7 @@ import java.nio.channels.Channel; import java.util.Collections; import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; import net.bytebuddy.asm.Advice; @AutoService(InstrumenterModule.class) @@ -66,7 +67,14 @@ public static final class Construct { public static void after(@Advice.This Object command) { ContextStore contextStore = InstrumentationContext.get(QUEUED_COMMAND, STATE); capture(contextStore, command); - QueueTimerHelper.startQueuingTimer(contextStore, Channel.class, command); + // FIXME hard to handle both the lifecyle and get access to the queue instance in the same + // frame within the WriteQueue class. + // This means we can't get the queue length. A (bad) alternative would be to instrument + // ConcurrentLinkedQueue broadly, + // or we could write more brittle instrumentation targeting code patterns in different gRPC + // versions. + QueueTimerHelper.startQueuingTimer( + contextStore, Channel.class, ConcurrentLinkedQueue.class, 0, command); } } diff --git a/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/executor/ThreadPoolExecutorInstrumentation.java b/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/executor/ThreadPoolExecutorInstrumentation.java index 6209990b6a1..229d22ec42b 100644 --- a/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/executor/ThreadPoolExecutorInstrumentation.java +++ b/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/executor/ThreadPoolExecutorInstrumentation.java @@ -31,6 +31,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Queue; import java.util.concurrent.RunnableFuture; import java.util.concurrent.ThreadPoolExecutor; import net.bytebuddy.asm.Advice; @@ -162,12 +163,20 @@ public static void capture( // excluded as // Runnables but it is not until now that they will be put on the executor's queue if (!exclude(RUNNABLE, task)) { + Queue queue = tpe.getQueue(); QueueTimerHelper.startQueuingTimer( - InstrumentationContext.get(Runnable.class, State.class), tpe.getClass(), task); + InstrumentationContext.get(Runnable.class, State.class), + tpe.getClass(), + queue.getClass(), + queue.size(), + task); } else if (!exclude(RUNNABLE_FUTURE, task) && task instanceof RunnableFuture) { + Queue queue = tpe.getQueue(); QueueTimerHelper.startQueuingTimer( InstrumentationContext.get(RunnableFuture.class, State.class), tpe.getClass(), + queue.getClass(), + queue.size(), (RunnableFuture) task); } } diff --git a/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/forkjoin/JavaForkJoinPoolInstrumentation.java b/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/forkjoin/JavaForkJoinPoolInstrumentation.java index 16da6df78f3..24d39e04168 100644 --- a/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/forkjoin/JavaForkJoinPoolInstrumentation.java +++ b/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/forkjoin/JavaForkJoinPoolInstrumentation.java @@ -16,10 +16,8 @@ import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.bootstrap.ContextStore; import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.bootstrap.instrumentation.java.concurrent.QueueTimerHelper; import datadog.trace.bootstrap.instrumentation.java.concurrent.State; import java.util.Map; -import java.util.concurrent.ForkJoinPool; import java.util.concurrent.ForkJoinTask; import net.bytebuddy.asm.Advice; @@ -53,13 +51,11 @@ public void methodAdvice(MethodTransformer transformer) { public static final class ExternalPush { @SuppressWarnings("rawtypes") @Advice.OnMethodEnter - public static void externalPush( - @Advice.This ForkJoinPool pool, @Advice.Argument(0) ForkJoinTask task) { + public static void externalPush(@Advice.Argument(0) ForkJoinTask task) { if (!exclude(FORK_JOIN_TASK, task)) { ContextStore contextStore = InstrumentationContext.get(ForkJoinTask.class, State.class); capture(contextStore, task); - QueueTimerHelper.startQueuingTimer(contextStore, pool.getClass(), task); } } @@ -74,13 +70,11 @@ public static void cleanup( public static final class PoolSubmit { @Advice.OnMethodEnter - public static void poolSubmit( - @Advice.This ForkJoinPool pool, @Advice.Argument(1) ForkJoinTask task) { + public static void poolSubmit(@Advice.Argument(1) ForkJoinTask task) { if (!exclude(FORK_JOIN_TASK, task)) { ContextStore contextStore = InstrumentationContext.get(ForkJoinTask.class, State.class); capture(contextStore, task); - QueueTimerHelper.startQueuingTimer(contextStore, pool.getClass(), task); } } diff --git a/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/forkjoin/JavaForkJoinWorkQueueInstrumentation.java b/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/forkjoin/JavaForkJoinWorkQueueInstrumentation.java new file mode 100644 index 00000000000..3be741c9908 --- /dev/null +++ b/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/forkjoin/JavaForkJoinWorkQueueInstrumentation.java @@ -0,0 +1,89 @@ +package datadog.trace.instrumentation.java.concurrent.forkjoin; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.declaresField; +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static datadog.trace.bootstrap.instrumentation.java.concurrent.ExcludeFilter.ExcludeType.FORK_JOIN_TASK; +import static datadog.trace.bootstrap.instrumentation.java.concurrent.ExcludeFilter.exclude; +import static datadog.trace.instrumentation.java.concurrent.ConcurrentInstrumentationNames.EXECUTOR_INSTRUMENTATION_NAME; +import static datadog.trace.instrumentation.java.concurrent.ConcurrentInstrumentationNames.FORK_JOIN_POOL_INSTRUMENTATION_NAME; +import static java.util.Collections.singletonMap; +import static net.bytebuddy.matcher.ElementMatchers.fieldType; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.api.config.ProfilingConfig; +import datadog.trace.bootstrap.ContextStore; +import datadog.trace.bootstrap.InstrumentationContext; +import datadog.trace.bootstrap.config.provider.ConfigProvider; +import datadog.trace.bootstrap.instrumentation.java.concurrent.QueueTimerHelper; +import datadog.trace.bootstrap.instrumentation.java.concurrent.State; +import java.util.Map; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinTask; +import net.bytebuddy.asm.Advice; + +@AutoService(InstrumenterModule.class) +public class JavaForkJoinWorkQueueInstrumentation extends InstrumenterModule.Profiling + implements Instrumenter.ForBootstrap, Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + + public JavaForkJoinWorkQueueInstrumentation() { + super( + EXECUTOR_INSTRUMENTATION_NAME, + FORK_JOIN_POOL_INSTRUMENTATION_NAME, + FORK_JOIN_POOL_INSTRUMENTATION_NAME + "-workqueue"); + } + + @Override + public String instrumentedType() { + return "java.util.concurrent.ForkJoinPool$WorkQueue"; + } + + @Override + public boolean isEnabled() { + return super.isEnabled() + && ConfigProvider.getInstance() + .getBoolean( + ProfilingConfig.PROFILING_QUEUEING_TIME_ENABLED, + ProfilingConfig.PROFILING_QUEUEING_TIME_ENABLED_DEFAULT); + } + + @Override + public Map contextStore() { + return singletonMap("java.util.concurrent.ForkJoinTask", State.class.getName()); + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + String name = getClass().getName(); + transformer.applyAdvice( + isMethod() + .and(named("push")) + .and(takesArgument(0, named("java.util.concurrent.ForkJoinTask"))) + .and( + isDeclaredBy( + declaresField(fieldType(int.class).and(named("top"))) + .and(declaresField(fieldType(int.class).and(named("base")))))), + name + "$PushTask"); + } + + public static final class PushTask { + @SuppressWarnings("rawtypes") + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void push( + @Advice.This Object workQueue, + @Advice.FieldValue("top") int top, + @Advice.FieldValue("base") int base, + @Advice.Argument(0) ForkJoinTask task) { + if (!exclude(FORK_JOIN_TASK, task)) { + ContextStore contextStore = + InstrumentationContext.get(ForkJoinTask.class, State.class); + QueueTimerHelper.startQueuingTimer( + contextStore, ForkJoinPool.class, workQueue.getClass(), top - base, task); + } + } + } +} diff --git a/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/timer/JavaTimerInstrumentation.java b/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/timer/JavaTimerInstrumentation.java index 9f5da393811..820da7bb54a 100644 --- a/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/timer/JavaTimerInstrumentation.java +++ b/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/timer/JavaTimerInstrumentation.java @@ -5,7 +5,6 @@ import static datadog.trace.bootstrap.instrumentation.java.concurrent.AdviceUtils.capture; import static datadog.trace.bootstrap.instrumentation.java.concurrent.ExcludeFilter.ExcludeType.RUNNABLE; import static datadog.trace.bootstrap.instrumentation.java.concurrent.ExcludeFilter.exclude; -import static datadog.trace.bootstrap.instrumentation.java.concurrent.QueueTimerHelper.startQueuingTimer; import static datadog.trace.instrumentation.java.concurrent.ConcurrentInstrumentationNames.EXECUTOR_INSTRUMENTATION_NAME; import static datadog.trace.instrumentation.java.concurrent.ConcurrentInstrumentationNames.RUNNABLE_INSTRUMENTATION_NAME; import static java.util.Collections.singletonMap; @@ -21,7 +20,6 @@ import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.java.concurrent.State; import java.util.Map; -import java.util.Timer; import java.util.TimerTask; import net.bytebuddy.asm.Advice; @@ -67,7 +65,6 @@ public static void before(@Advice.Argument(0) TimerTask task, @Advice.Argument(2 ContextStore contextStore = InstrumentationContext.get(Runnable.class, State.class); capture(contextStore, task); - startQueuingTimer(contextStore, Timer.class, task); } } diff --git a/dd-java-agent/instrumentation/java-concurrent/src/test/groovy/QueueTimingForkedTest.groovy b/dd-java-agent/instrumentation/java-concurrent/src/test/groovy/QueueTimingForkedTest.groovy index bf74c185d5f..dc7369692e7 100644 --- a/dd-java-agent/instrumentation/java-concurrent/src/test/groovy/QueueTimingForkedTest.groovy +++ b/dd-java-agent/instrumentation/java-concurrent/src/test/groovy/QueueTimingForkedTest.groovy @@ -1,8 +1,11 @@ import datadog.trace.agent.test.AgentTestRunner import datadog.trace.agent.test.TestProfilingContextIntegration +import datadog.trace.api.Platform import datadog.trace.bootstrap.instrumentation.jfr.InstrumentationBasedProfiling import java.util.concurrent.Executors +import java.util.concurrent.ForkJoinPool +import java.util.concurrent.LinkedBlockingQueue import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace @@ -20,6 +23,7 @@ class QueueTimingForkedTest extends AgentTestRunner { def "test queue timing with submit"() { setup: def executor = Executors.newSingleThreadExecutor() + def fjp = new ForkJoinPool(1) when: runUnderTrace("parent", { @@ -27,14 +31,26 @@ class QueueTimingForkedTest extends AgentTestRunner { }) then: - verify() + verify(LinkedBlockingQueue.name) + + when: + runUnderTrace("parent", { + fjp.submit(new TestRunnable()).get() + }) + + then: + // flaky before JDK21 + if (Platform.isJavaVersionAtLeast(21)) { + verify("java.util.concurrent.ForkJoinPool\$WorkQueue") + } cleanup: executor.shutdown() + fjp.shutdown() TEST_PROFILING_CONTEXT_INTEGRATION.closedTimings.clear() } - void verify() { + void verify(expectedQueueType) { assert TEST_PROFILING_CONTEXT_INTEGRATION.isBalanced() assert !TEST_PROFILING_CONTEXT_INTEGRATION.closedTimings.isEmpty() int numAsserts = 0 @@ -45,6 +61,8 @@ class QueueTimingForkedTest extends AgentTestRunner { assert timing.task == TestRunnable assert timing.scheduler != null assert timing.origin == Thread.currentThread() + assert timing.queueLength >= 0 + assert timing.queue.name == expectedQueueType numAsserts++ } } diff --git a/dd-java-agent/instrumentation/netty-concurrent-4/src/main/java/datadog/trace/instrumentation/netty40/concurrent/SingleThreadEventExecutorInstrumentation.java b/dd-java-agent/instrumentation/netty-concurrent-4/src/main/java/datadog/trace/instrumentation/netty40/concurrent/SingleThreadEventExecutorInstrumentation.java index 0fa3505b4cb..a9779e28b87 100644 --- a/dd-java-agent/instrumentation/netty-concurrent-4/src/main/java/datadog/trace/instrumentation/netty40/concurrent/SingleThreadEventExecutorInstrumentation.java +++ b/dd-java-agent/instrumentation/netty-concurrent-4/src/main/java/datadog/trace/instrumentation/netty40/concurrent/SingleThreadEventExecutorInstrumentation.java @@ -1,7 +1,9 @@ package datadog.trace.instrumentation.netty40.concurrent; +import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.declaresField; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.nameEndsWith; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; import static net.bytebuddy.matcher.ElementMatchers.isMethod; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; import static net.bytebuddy.matcher.ElementMatchers.takesArguments; @@ -19,6 +21,7 @@ import io.netty.util.concurrent.EventExecutorGroup; import java.util.Collections; import java.util.Map; +import java.util.Queue; import java.util.concurrent.RunnableFuture; import net.bytebuddy.asm.Advice; @@ -47,9 +50,7 @@ public Map contextStore() { public String[] knownMatchingTypes() { return new String[] { "io.netty.util.concurrent.SingleThreadEventExecutor", - "io.netty.util.concurrent.AbstractScheduledEventExecutor", "io.grpc.shaded.io.netty.util.concurrent.SingleThreadEventExecutor", - "io.grpc.shaded.io.netty.util.concurrent.AbstractScheduledEventExecutor" }; } @@ -59,23 +60,84 @@ public void methodAdvice(MethodTransformer transformer) { isMethod() .and(named("addTask")) .and(takesArguments(1)) - .and(takesArgument(0, Runnable.class)), - getClass().getName() + "$StartTiming"); - // schedule may call execute so using the same instrumentation relies on detecting double - // timing - earliest (schedule) must take precedence + .and(takesArgument(0, Runnable.class)) + .and(isDeclaredBy(declaresField(named("taskQueue")))), + getClass().getName() + "$StartTimingTaskQueue"); transformer.applyAdvice( isMethod() .and(named("schedule")) .and(takesArguments(1)) - .and(takesArgument(0, nameEndsWith("netty.util.concurrent.ScheduledFutureTask"))), - getClass().getName() + "$StartTiming"); + .and(takesArgument(0, nameEndsWith("netty.util.concurrent.ScheduledFutureTask"))) + .and(isDeclaredBy(declaresField(named("delayedTaskQueue")))), + getClass().getName() + "$StartTimingDelayedTaskQueue"); + transformer.applyAdvice( + isMethod() + .and(named("schedule")) + .and(takesArguments(1)) + .and(takesArgument(0, nameEndsWith("netty.util.concurrent.ScheduledFutureTask"))) + .and(isDeclaredBy(declaresField(named("scheduledTaskQueue")))), + getClass().getName() + "$StartTimingScheduledTaskQueue"); + } + + @SuppressWarnings("rawtypes") + private static final class StartTimingTaskQueue { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void before( + @Advice.FieldValue("taskQueue") Queue taskQueue, + @Advice.This EventExecutor executor, + @Advice.Argument(0) Runnable task) { + // should be a PromiseTask which is a RunnableFuture + if (task instanceof RunnableFuture) { + // detect double timing - also not interested in queueing time unless the task is traced + // state == null => not traced, state.isTimed() => double timing + ContextStore contextStore = + InstrumentationContext.get(RunnableFuture.class, State.class); + State state = contextStore.get((RunnableFuture) task); + if (state == null || state.isTimed() || taskQueue == null) { + return; + } + Class queueType = taskQueue == null ? null : taskQueue.getClass(); + int length = taskQueue == null ? 0 : taskQueue.size(); + EventExecutorGroup parent = executor.parent(); + Class schedulerClass = parent == null ? executor.getClass() : parent.getClass(); + QueueTimerHelper.startQueuingTimer(state, schedulerClass, queueType, length, task); + } + } + } + + @SuppressWarnings("rawtypes") + private static final class StartTimingDelayedTaskQueue { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void before( + @Advice.FieldValue("delayedTaskQueue") Queue delayedTaskQueue, + @Advice.This EventExecutor executor, + @Advice.Argument(0) Runnable task) { + // should be a PromiseTask which is a RunnableFuture + if (task instanceof RunnableFuture) { + // detect double timing - also not interested in queueing time unless the task is traced + // state == null => not traced, state.isTimed() => double timing + ContextStore contextStore = + InstrumentationContext.get(RunnableFuture.class, State.class); + State state = contextStore.get((RunnableFuture) task); + if (state == null || state.isTimed()) { + return; + } + Class queueType = delayedTaskQueue == null ? null : delayedTaskQueue.getClass(); + int length = delayedTaskQueue == null ? 0 : delayedTaskQueue.size(); + EventExecutorGroup parent = executor.parent(); + Class schedulerClass = parent == null ? executor.getClass() : parent.getClass(); + QueueTimerHelper.startQueuingTimer(state, schedulerClass, queueType, length, task); + } + } } @SuppressWarnings("rawtypes") - private static final class StartTiming { + private static final class StartTimingScheduledTaskQueue { @Advice.OnMethodEnter(suppress = Throwable.class) public static void before( - @Advice.This EventExecutor executor, @Advice.Argument(0) Runnable task) { + @Advice.FieldValue("scheduledTaskQueue") Queue scheduledTaskQueue, + @Advice.This EventExecutor executor, + @Advice.Argument(0) Runnable task) { // should be a PromiseTask which is a RunnableFuture if (task instanceof RunnableFuture) { // detect double timing - also not interested in queueing time unless the task is traced @@ -86,9 +148,11 @@ public static void before( if (state == null || state.isTimed()) { return; } + Class queueType = scheduledTaskQueue == null ? null : scheduledTaskQueue.getClass(); + int length = scheduledTaskQueue == null ? 0 : scheduledTaskQueue.size(); EventExecutorGroup parent = executor.parent(); Class schedulerClass = parent == null ? executor.getClass() : parent.getClass(); - QueueTimerHelper.startQueuingTimer(state, schedulerClass, task); + QueueTimerHelper.startQueuingTimer(state, schedulerClass, queueType, length, task); } } } diff --git a/dd-java-agent/instrumentation/netty-concurrent-4/src/test/groovy/TimingTest.groovy b/dd-java-agent/instrumentation/netty-concurrent-4/src/test/groovy/TimingTest.groovy index 3364b67fb20..e5a5fcf561f 100644 --- a/dd-java-agent/instrumentation/netty-concurrent-4/src/test/groovy/TimingTest.groovy +++ b/dd-java-agent/instrumentation/netty-concurrent-4/src/test/groovy/TimingTest.groovy @@ -3,6 +3,7 @@ import datadog.trace.agent.test.TestProfilingContextIntegration import datadog.trace.bootstrap.instrumentation.jfr.InstrumentationBasedProfiling import io.netty.util.concurrent.DefaultEventExecutorGroup +import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace @@ -27,7 +28,7 @@ class TimingTest extends AgentTestRunner { }) then: - verify() + verify([LinkedBlockingQueue]) cleanup: defaultEventExecutorGroup.shutdownGracefully() @@ -44,14 +45,15 @@ class TimingTest extends AgentTestRunner { }) then: - verify() + // later netty versions may schedule on the task queue, older versions on the scheduled/delayed queue + verify([PriorityQueue, LinkedBlockingQueue]) cleanup: defaultEventExecutorGroup.shutdownGracefully() TEST_PROFILING_CONTEXT_INTEGRATION.closedTimings.clear() } - void verify() { + void verify(acceptableQueueTypes) { assert TEST_PROFILING_CONTEXT_INTEGRATION.isBalanced() assert !TEST_PROFILING_CONTEXT_INTEGRATION.closedTimings.isEmpty() int numAsserts = 0 @@ -64,6 +66,8 @@ class TimingTest extends AgentTestRunner { assert timing.task == TestRunnable assert timing.scheduler == DefaultEventExecutorGroup assert timing.origin == Thread.currentThread() + assert timing.queueLength == 0 + assert acceptableQueueTypes.contains(timing.queue) numAsserts++ } } diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy index 0803d35cfdd..370646949e7 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/TestProfilingContextIntegration.groovy @@ -79,6 +79,8 @@ class TestProfilingContextIntegration implements ProfilingContextIntegration { Class task Class scheduler + Class queue + int queueLength final Thread origin final long start @@ -98,6 +100,16 @@ class TestProfilingContextIntegration implements ProfilingContextIntegration { this.scheduler = scheduler } + @Override + void setQueue(Class queue) { + this.queue = queue + } + + @Override + void setQueueLength(int queueLength) { + this.queueLength = queueLength + } + @Override void report() { counter.decrementAndGet() diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 73914c115c9..292c478f00c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -29,7 +29,7 @@ moshi = '1.11.0' testcontainers = '1.20.1' jmc = "8.1.0" autoservice = "1.0-rc7" -ddprof = "1.18.0" +ddprof = "1.19.0" asm = "9.7.1" cafe_crypto = "0.1.0" lz4 = "1.7.1" diff --git a/internal-api/src/main/java/datadog/trace/api/profiling/QueueTiming.java b/internal-api/src/main/java/datadog/trace/api/profiling/QueueTiming.java index 082f5a8be9c..bdf397d0c09 100644 --- a/internal-api/src/main/java/datadog/trace/api/profiling/QueueTiming.java +++ b/internal-api/src/main/java/datadog/trace/api/profiling/QueueTiming.java @@ -5,4 +5,8 @@ public interface QueueTiming extends Timing { void setTask(Object task); void setScheduler(Class scheduler); + + void setQueue(Class queue); + + void setQueueLength(int queueLength); } diff --git a/internal-api/src/main/java/datadog/trace/api/profiling/Timing.java b/internal-api/src/main/java/datadog/trace/api/profiling/Timing.java index 7e87ce95ef8..dc3af48860b 100644 --- a/internal-api/src/main/java/datadog/trace/api/profiling/Timing.java +++ b/internal-api/src/main/java/datadog/trace/api/profiling/Timing.java @@ -21,5 +21,11 @@ public void setTask(Object task) {} @Override public void setScheduler(Class scheduler) {} + + @Override + public void setQueue(Class queue) {} + + @Override + public void setQueueLength(int queueLength) {} } }