[WIP] KAFKA-18376: High CPU load when AsyncKafkaConsumer uses a small…

… max poll value Introduces CompositePollEvent and CompositePollResult to refactor and streamline the poll event handling in AsyncKafkaConsumer and ApplicationEventProcessor. The new approach enables multi-step polling logic, improves callback and background event processing, and enhances testability. Also adds size methods to BackgroundEventHandler and OffsetCommitCallbackInvoker, disables several tests, and updates related classes to support the new event flow.
apache · kirktrue · Sep 10, 2025 · Sep 11, 2025 · Sep 11, 2025 · Sep 15, 2025
commit 34932e22224f7c108959da155c7069d141c56972
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AsyncKafkaConsumer.java
@@ -49,6 +49,8 @@
 import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent;
 import org.apache.kafka.clients.consumer.internals.events.CompletableEvent;
 import org.apache.kafka.clients.consumer.internals.events.CompletableEventReaper;
+import org.apache.kafka.clients.consumer.internals.events.CompositePollEvent;
+import org.apache.kafka.clients.consumer.internals.events.CompositePollResult;
 import org.apache.kafka.clients.consumer.internals.events.ConsumerRebalanceListenerCallbackCompletedEvent;
 import org.apache.kafka.clients.consumer.internals.events.ConsumerRebalanceListenerCallbackNeededEvent;
 import org.apache.kafka.clients.consumer.internals.events.CreateFetchRequestsEvent;
@@ -59,7 +61,6 @@
 import org.apache.kafka.clients.consumer.internals.events.LeaveGroupOnCloseEvent;
 import org.apache.kafka.clients.consumer.internals.events.ListOffsetsEvent;
 import org.apache.kafka.clients.consumer.internals.events.PausePartitionsEvent;
-import org.apache.kafka.clients.consumer.internals.events.PollEvent;
 import org.apache.kafka.clients.consumer.internals.events.ResetOffsetEvent;
 import org.apache.kafka.clients.consumer.internals.events.ResumePartitionsEvent;
 import org.apache.kafka.clients.consumer.internals.events.SeekUnvalidatedEvent;
@@ -500,7 +501,10 @@ public AsyncKafkaConsumer(final ConsumerConfig config,
             final Supplier<ApplicationEventProcessor> applicationEventProcessorSupplier = ApplicationEventProcessor.supplier(logContext,
                     metadata,
                     subscriptions,
-                    requestManagersSupplier);
+                    requestManagersSupplier,
+                    backgroundEventHandler,
+                    Optional.of(offsetCommitCallbackInvoker)
+            );
             this.applicationEventHandler = applicationEventHandlerFactory.build(
                     logContext,
                     time,
@@ -689,7 +693,9 @@ public AsyncKafkaConsumer(final ConsumerConfig config,
                 logContext,
                 metadata,
                 subscriptions,
-                requestManagersSupplier
+                requestManagersSupplier,
+                backgroundEventHandler,
+                Optional.of(offsetCommitCallbackInvoker)
         );
         this.applicationEventHandler = new ApplicationEventHandler(logContext,
                 time,
@@ -865,23 +871,32 @@ public ConsumerRecords<K, V> poll(final Duration timeout) {
             }
 
             do {
-                PollEvent event = new PollEvent(timer.currentTimeMs());
-                // Make sure to let the background thread know that we are still polling.
-                // This will trigger async auto-commits of consumed positions when hitting
-                // the interval time or reconciling new assignments
-                applicationEventHandler.add(event);
-                // Wait for reconciliation and auto-commit to be triggered, to ensure all commit requests
-                // retrieve the positions to commit before proceeding with fetching new records
-                ConsumerUtils.getResult(event.reconcileAndAutoCommit(), defaultApiTimeoutMs.toMillis());
-
                 // We must not allow wake-ups between polling for fetches and returning the records.
                 // If the polled fetches are not empty the consumed position has already been updated in the polling
                 // of the fetches. A wakeup between returned fetches and returning records would lead to never
                 // returning the records in the fetches. Thus, we trigger a possible wake-up before we poll fetches.
                 wakeupTrigger.maybeTriggerWakeup();
 
-                updateAssignmentMetadataIfNeeded(timer);
-                final Fetch<K, V> fetch = pollForFetches(timer);
+                long pollTimeMs = timer.currentTimeMs();
+                long deadlineMs = calculateDeadlineMs(timer);
+                ApplicationEvent.Type nextStep = ApplicationEvent.Type.POLL;
+
+                while (true) {
+                    CompositePollEvent event = new CompositePollEvent(pollTimeMs, deadlineMs, nextStep);
+                    CompositePollResult result = applicationEventHandler.addAndGet(event);
+
+                    if (result == CompositePollResult.NEEDS_OFFSET_COMMIT_CALLBACKS) {
+                        offsetCommitCallbackInvoker.executeCallbacks();
+                        nextStep = ApplicationEvent.Type.UPDATE_SUBSCRIPTION_METADATA;
+                    } else if (result == CompositePollResult.NEEDS_BACKGROUND_EVENT_PROCESSING) {
+                        processBackgroundEvents();
+                        nextStep = ApplicationEvent.Type.CHECK_AND_UPDATE_POSITIONS;
+                    } else if (result == CompositePollResult.COMPLETE) {
+                        break;
+                    }
+                }
+
+                final Fetch<K, V> fetch = collectFetch();
                 if (!fetch.isEmpty()) {
                     // before returning the fetched records, we can send off the next round of fetches
                     // and avoid block waiting for their responses to enable pipelining while the user

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java
@@ -77,6 +77,9 @@ protected void maybeThrowAuthFailure(Node node) {
      * @return Future on which the caller can wait to ensure that the requests have been created
      */
     public CompletableFuture<Void> createFetchRequests() {
+        if (!fetchBuffer.isEmpty())
+            return CompletableFuture.completedFuture(null);
+
         CompletableFuture<Void> future = new CompletableFuture<>();
 
         if (pendingFetchRequestFuture != null) {

diff --git a/...rc/main/java/org/apache/kafka/clients/consumer/internals/OffsetCommitCallbackInvoker.java b/...rc/main/java/org/apache/kafka/clients/consumer/internals/OffsetCommitCallbackInvoker.java
@@ -50,6 +50,10 @@ public void enqueueInterceptorInvocation(final Map<TopicPartition, OffsetAndMeta
         }
     }
 
+    public int size() {
+        return callbackQueue.size();
+    }
+
     public void enqueueUserCallbackInvocation(final OffsetCommitCallback callback,
                                               final Map<TopicPartition, OffsetAndMetadata> offsets,
                                               final Exception exception) {

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ShareConsumerImpl.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ShareConsumerImpl.java
@@ -301,7 +301,9 @@ private void process(final ShareAcknowledgementCommitCallbackEvent event) {
                     logContext,
                     metadata,
                     subscriptions,
-                    requestManagersSupplier
+                    requestManagersSupplier,
+                    backgroundEventHandler,
+                    Optional.empty()
             );
 
             this.applicationEventHandler = applicationEventHandlerFactory.build(
@@ -407,7 +409,9 @@ private void process(final ShareAcknowledgementCommitCallbackEvent event) {
                 logContext,
                 metadata,
                 subscriptions,
-                requestManagersSupplier
+                requestManagersSupplier,
+                backgroundEventHandler,
+                Optional.empty()
         );
 
         this.applicationEventHandler = new ApplicationEventHandler(

diff --git a/...ts/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java b/...ts/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java
@@ -28,7 +28,7 @@
 public abstract class ApplicationEvent {
 
     public enum Type {
-        COMMIT_ASYNC, COMMIT_SYNC, POLL, FETCH_COMMITTED_OFFSETS, NEW_TOPICS_METADATA_UPDATE, ASSIGNMENT_CHANGE,
+        COMMIT_ASYNC, COMMIT_SYNC, COMPOSITE_POLL, POLL, FETCH_COMMITTED_OFFSETS, NEW_TOPICS_METADATA_UPDATE, ASSIGNMENT_CHANGE,
         LIST_OFFSETS, CHECK_AND_UPDATE_POSITIONS, RESET_OFFSET, TOPIC_METADATA, ALL_TOPICS_METADATA,
         TOPIC_SUBSCRIPTION_CHANGE, TOPIC_PATTERN_SUBSCRIPTION_CHANGE, TOPIC_RE2J_PATTERN_SUBSCRIPTION_CHANGE,
         UPDATE_SUBSCRIPTION_METADATA, UNSUBSCRIBE,