From 3c1010b9bede7e18fc536749950eeb5d3cb567f7 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Wed, 26 Feb 2025 17:13:49 -0800 Subject: [PATCH 1/6] Rename GroupCoordinatorRequest/Response -> FindCoordinatorRequest/Response; move module --- kafka/admin/client.py | 9 +++--- kafka/cluster.py | 8 ++--- kafka/conn.py | 5 +-- kafka/coordinator/base.py | 5 +-- kafka/protocol/commit.py | 46 --------------------------- kafka/protocol/find_coordinator.py | 50 ++++++++++++++++++++++++++++++ kafka/protocol/parser.py | 4 +-- test/test_protocol.py | 4 +-- 8 files changed, 69 insertions(+), 62 deletions(-) create mode 100644 kafka/protocol/find_coordinator.py diff --git a/kafka/admin/client.py b/kafka/admin/client.py index 68b0af115..136c25854 100644 --- a/kafka/admin/client.py +++ b/kafka/admin/client.py @@ -23,7 +23,8 @@ ListGroupsRequest, DescribeGroupsRequest, DescribeAclsRequest, CreateAclsRequest, DeleteAclsRequest, DeleteGroupsRequest, DescribeLogDirsRequest ) -from kafka.protocol.commit import GroupCoordinatorRequest, OffsetFetchRequest +from kafka.protocol.commit import OffsetFetchRequest +from kafka.protocol.find_coordinator import FindCoordinatorRequest from kafka.protocol.metadata import MetadataRequest from kafka.protocol.types import Array from kafka.structs import TopicPartition, OffsetAndMetadata, MemberInformation, GroupInformation @@ -290,12 +291,12 @@ def _find_coordinator_id_send_request(self, group_id): # When I experimented with this, the coordinator value returned in # GroupCoordinatorResponse_v1 didn't match the value returned by # GroupCoordinatorResponse_v0 and I couldn't figure out why. - version = self._client.api_version(GroupCoordinatorRequest, max_version=0) + version = self._client.api_version(FindCoordinatorRequest, max_version=0) if version <= 0: - request = GroupCoordinatorRequest[version](group_id) + request = FindCoordinatorRequest[version](group_id) else: raise NotImplementedError( - "Support for GroupCoordinatorRequest_v{} has not yet been added to KafkaAdminClient." + "Support for FindCoordinatorRequest_v{} has not yet been added to KafkaAdminClient." .format(version)) return self._send_request_to_node(self._client.least_loaded_node(), request) diff --git a/kafka/cluster.py b/kafka/cluster.py index 69a49de07..98272ea1e 100644 --- a/kafka/cluster.py +++ b/kafka/cluster.py @@ -21,7 +21,7 @@ class ClusterMetadata(object): A class to manage kafka cluster metadata. This class does not perform any IO. It simply updates internal state - given API responses (MetadataResponse, GroupCoordinatorResponse). + given API responses (MetadataResponse, FindCoordinatorResponse). Keyword Arguments: retry_backoff_ms (int): Milliseconds to backoff when retrying on @@ -367,8 +367,8 @@ def add_group_coordinator(self, group, response): """Update with metadata for a group coordinator Arguments: - group (str): name of group from GroupCoordinatorRequest - response (GroupCoordinatorResponse): broker response + group (str): name of group from FindCoordinatorRequest + response (FindCoordinatorResponse): broker response Returns: string: coordinator node_id if metadata is updated, None on error @@ -376,7 +376,7 @@ def add_group_coordinator(self, group, response): log.debug("Updating coordinator for %s: %s", group, response) error_type = Errors.for_code(response.error_code) if error_type is not Errors.NoError: - log.error("GroupCoordinatorResponse error: %s", error_type) + log.error("FindCoordinatorResponse error: %s", error_type) self._groups[group] = -1 return diff --git a/kafka/conn.py b/kafka/conn.py index 347e5000b..6240e1530 100644 --- a/kafka/conn.py +++ b/kafka/conn.py @@ -1255,14 +1255,15 @@ def reset_override_configs(): from kafka.protocol.admin import ListGroupsRequest from kafka.protocol.api_versions import ApiVersionsRequest from kafka.protocol.broker_api_versions import BROKER_API_VERSIONS - from kafka.protocol.commit import OffsetFetchRequest, GroupCoordinatorRequest + from kafka.protocol.commit import OffsetFetchRequest + from kafka.protocol.find_coordinator import FindCoordinatorRequest test_cases = [ # All cases starting from 0.10 will be based on ApiVersionsResponse ((0, 11), ApiVersionsRequest[1]()), ((0, 10, 0), ApiVersionsRequest[0]()), ((0, 9), ListGroupsRequest[0]()), - ((0, 8, 2), GroupCoordinatorRequest[0]('kafka-python-default-group')), + ((0, 8, 2), FindCoordinatorRequest[0]('kafka-python-default-group')), ((0, 8, 1), OffsetFetchRequest[0]('kafka-python-default-group', [])), ((0, 8, 0), MetadataRequest[0](topics)), ] diff --git a/kafka/coordinator/base.py b/kafka/coordinator/base.py index d18de0743..accf99410 100644 --- a/kafka/coordinator/base.py +++ b/kafka/coordinator/base.py @@ -14,7 +14,8 @@ from kafka.future import Future from kafka.metrics import AnonMeasurable from kafka.metrics.stats import Avg, Count, Max, Rate -from kafka.protocol.commit import GroupCoordinatorRequest, OffsetCommitRequest +from kafka.protocol.commit import OffsetCommitRequest +from kafka.protocol.find_coordinator import FindCoordinatorRequest from kafka.protocol.group import (HeartbeatRequest, JoinGroupRequest, LeaveGroupRequest, SyncGroupRequest) @@ -660,7 +661,7 @@ def _send_group_coordinator_request(self): log.debug("Sending group coordinator request for group %s to broker %s", self.group_id, node_id) - request = GroupCoordinatorRequest[0](self.group_id) + request = FindCoordinatorRequest[0](self.group_id) future = Future() _f = self._client.send(node_id, request) _f.add_callback(self._handle_group_coordinator_response, future) diff --git a/kafka/protocol/commit.py b/kafka/protocol/commit.py index 31fc23707..f5828ba59 100644 --- a/kafka/protocol/commit.py +++ b/kafka/protocol/commit.py @@ -207,49 +207,3 @@ class OffsetFetchRequest_v3(Request): OffsetFetchResponse_v0, OffsetFetchResponse_v1, OffsetFetchResponse_v2, OffsetFetchResponse_v3, ] - - -class GroupCoordinatorResponse_v0(Response): - API_KEY = 10 - API_VERSION = 0 - SCHEMA = Schema( - ('error_code', Int16), - ('coordinator_id', Int32), - ('host', String('utf-8')), - ('port', Int32) - ) - - -class GroupCoordinatorResponse_v1(Response): - API_KEY = 10 - API_VERSION = 1 - SCHEMA = Schema( - ('error_code', Int16), - ('error_message', String('utf-8')), - ('coordinator_id', Int32), - ('host', String('utf-8')), - ('port', Int32) - ) - - -class GroupCoordinatorRequest_v0(Request): - API_KEY = 10 - API_VERSION = 0 - RESPONSE_TYPE = GroupCoordinatorResponse_v0 - SCHEMA = Schema( - ('consumer_group', String('utf-8')) - ) - - -class GroupCoordinatorRequest_v1(Request): - API_KEY = 10 - API_VERSION = 1 - RESPONSE_TYPE = GroupCoordinatorResponse_v1 - SCHEMA = Schema( - ('coordinator_key', String('utf-8')), - ('coordinator_type', Int8) - ) - - -GroupCoordinatorRequest = [GroupCoordinatorRequest_v0, GroupCoordinatorRequest_v1] -GroupCoordinatorResponse = [GroupCoordinatorResponse_v0, GroupCoordinatorResponse_v1] diff --git a/kafka/protocol/find_coordinator.py b/kafka/protocol/find_coordinator.py new file mode 100644 index 000000000..d93e2d39a --- /dev/null +++ b/kafka/protocol/find_coordinator.py @@ -0,0 +1,50 @@ +from __future__ import absolute_import + +from kafka.protocol.api import Request, Response +from kafka.protocol.types import Array, Int8, Int16, Int32, Int64, Schema, String + + +class FindCoordinatorResponse_v0(Response): + API_KEY = 10 + API_VERSION = 0 + SCHEMA = Schema( + ('error_code', Int16), + ('coordinator_id', Int32), + ('host', String('utf-8')), + ('port', Int32) + ) + + +class FindCoordinatorResponse_v1(Response): + API_KEY = 10 + API_VERSION = 1 + SCHEMA = Schema( + ('error_code', Int16), + ('error_message', String('utf-8')), + ('coordinator_id', Int32), + ('host', String('utf-8')), + ('port', Int32) + ) + + +class FindCoordinatorRequest_v0(Request): + API_KEY = 10 + API_VERSION = 0 + RESPONSE_TYPE = FindCoordinatorResponse_v0 + SCHEMA = Schema( + ('consumer_group', String('utf-8')) + ) + + +class FindCoordinatorRequest_v1(Request): + API_KEY = 10 + API_VERSION = 1 + RESPONSE_TYPE = FindCoordinatorResponse_v1 + SCHEMA = Schema( + ('coordinator_key', String('utf-8')), + ('coordinator_type', Int8) + ) + + +FindCoordinatorRequest = [FindCoordinatorRequest_v0, FindCoordinatorRequest_v1] +FindCoordinatorResponse = [FindCoordinatorResponse_v0, FindCoordinatorResponse_v1] diff --git a/kafka/protocol/parser.py b/kafka/protocol/parser.py index a9e767220..e7799fce6 100644 --- a/kafka/protocol/parser.py +++ b/kafka/protocol/parser.py @@ -4,7 +4,7 @@ import logging import kafka.errors as Errors -from kafka.protocol.commit import GroupCoordinatorResponse +from kafka.protocol.find_coordinator import FindCoordinatorResponse from kafka.protocol.frame import KafkaBytes from kafka.protocol.types import Int32, TaggedFields from kafka.version import __version__ @@ -142,7 +142,7 @@ def _process_response(self, read_buffer): # 0.8.2 quirk if (recv_correlation_id == 0 and correlation_id != 0 and - request.RESPONSE_TYPE is GroupCoordinatorResponse[0] and + request.RESPONSE_TYPE is FindCoordinatorResponse[0] and (self._api_version == (0, 8, 2) or self._api_version is None)): log.warning('Kafka 0.8.2 quirk -- GroupCoordinatorResponse' ' Correlation ID does not match request. This' diff --git a/test/test_protocol.py b/test/test_protocol.py index 6a77e19d6..6f94c74e1 100644 --- a/test/test_protocol.py +++ b/test/test_protocol.py @@ -5,8 +5,8 @@ import pytest from kafka.protocol.api import RequestHeader -from kafka.protocol.commit import GroupCoordinatorRequest from kafka.protocol.fetch import FetchRequest, FetchResponse +from kafka.protocol.find_coordinator import FindCoordinatorRequest from kafka.protocol.message import Message, MessageSet, PartialMessage from kafka.protocol.metadata import MetadataRequest from kafka.protocol.types import Int16, Int32, Int64, String, UnsignedVarInt32, CompactString, CompactArray, CompactBytes @@ -168,7 +168,7 @@ def test_encode_message_header(): b'client3', # ClientId ]) - req = GroupCoordinatorRequest[0]('foo') + req = FindCoordinatorRequest[0]('foo') header = RequestHeader(req, correlation_id=4, client_id='client3') assert header.encode() == expect From 1d13aed0ce4620d548afde8eb9afd40dc974ea9d Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Wed, 26 Feb 2025 17:14:16 -0800 Subject: [PATCH 2/6] fix FindCoordinatorRequest v1 throttle_time_ms; add comment re type --- kafka/protocol/find_coordinator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kafka/protocol/find_coordinator.py b/kafka/protocol/find_coordinator.py index d93e2d39a..05e3168bf 100644 --- a/kafka/protocol/find_coordinator.py +++ b/kafka/protocol/find_coordinator.py @@ -19,6 +19,7 @@ class FindCoordinatorResponse_v1(Response): API_KEY = 10 API_VERSION = 1 SCHEMA = Schema( + ('throttle_time_ms', Int32), ('error_code', Int16), ('error_message', String('utf-8')), ('coordinator_id', Int32), @@ -42,7 +43,7 @@ class FindCoordinatorRequest_v1(Request): RESPONSE_TYPE = FindCoordinatorResponse_v1 SCHEMA = Schema( ('coordinator_key', String('utf-8')), - ('coordinator_type', Int8) + ('coordinator_type', Int8) # 0: consumer, 1: transaction ) From 19c2d4e2c6f7ae2ec1b54a5a3d3ee1a4ef1e930b Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Wed, 26 Feb 2025 17:15:07 -0800 Subject: [PATCH 3/6] Add v2 FindCoordinatorRequest/Response --- kafka/protocol/find_coordinator.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/kafka/protocol/find_coordinator.py b/kafka/protocol/find_coordinator.py index 05e3168bf..a68a23902 100644 --- a/kafka/protocol/find_coordinator.py +++ b/kafka/protocol/find_coordinator.py @@ -28,6 +28,12 @@ class FindCoordinatorResponse_v1(Response): ) +class FindCoordinatorResponse_v2(Response): + API_KEY = 10 + API_VERSION = 2 + SCHEMA = FindCoordinatorResponse_v1.SCHEMA + + class FindCoordinatorRequest_v0(Request): API_KEY = 10 API_VERSION = 0 @@ -47,5 +53,12 @@ class FindCoordinatorRequest_v1(Request): ) -FindCoordinatorRequest = [FindCoordinatorRequest_v0, FindCoordinatorRequest_v1] -FindCoordinatorResponse = [FindCoordinatorResponse_v0, FindCoordinatorResponse_v1] +class FindCoordinatorRequest_v2(Request): + API_KEY = 10 + API_VERSION = 2 + RESPONSE_TYPE = FindCoordinatorResponse_v2 + SCHEMA = FindCoordinatorRequest_v1.SCHEMA + + +FindCoordinatorRequest = [FindCoordinatorRequest_v0, FindCoordinatorRequest_v1, FindCoordinatorRequest_v2] +FindCoordinatorResponse = [FindCoordinatorResponse_v0, FindCoordinatorResponse_v1, FindCoordinatorResponse_v2] From 22e669dfd7c2c87022e81db4e30ce933b0300733 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Wed, 26 Feb 2025 17:17:13 -0800 Subject: [PATCH 4/6] Use FindCoordinatorRequest v2 in consumer and admin --- kafka/admin/client.py | 9 +++------ kafka/coordinator/base.py | 9 ++++++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/kafka/admin/client.py b/kafka/admin/client.py index 136c25854..10874444c 100644 --- a/kafka/admin/client.py +++ b/kafka/admin/client.py @@ -286,14 +286,11 @@ def _find_coordinator_id_send_request(self, group_id): Returns: A message future """ - # TODO add support for dynamically picking version of - # GroupCoordinatorRequest which was renamed to FindCoordinatorRequest. - # When I experimented with this, the coordinator value returned in - # GroupCoordinatorResponse_v1 didn't match the value returned by - # GroupCoordinatorResponse_v0 and I couldn't figure out why. - version = self._client.api_version(FindCoordinatorRequest, max_version=0) + version = self._client.api_version(FindCoordinatorRequest, max_version=2) if version <= 0: request = FindCoordinatorRequest[version](group_id) + elif version <= 2: + request = FindCoordinatorRequest[version](group_id, 0) else: raise NotImplementedError( "Support for FindCoordinatorRequest_v{} has not yet been added to KafkaAdminClient." diff --git a/kafka/coordinator/base.py b/kafka/coordinator/base.py index accf99410..354663a54 100644 --- a/kafka/coordinator/base.py +++ b/kafka/coordinator/base.py @@ -661,7 +661,11 @@ def _send_group_coordinator_request(self): log.debug("Sending group coordinator request for group %s to broker %s", self.group_id, node_id) - request = FindCoordinatorRequest[0](self.group_id) + version = self._client.api_version(FindCoordinatorRequest, max_version=2) + if version == 0: + request = FindCoordinatorRequest[0](self.group_id) + else: + request = FindCoordinatorRequest[0](self.group_id, 0) future = Future() _f = self._client.send(node_id, request) _f.add_callback(self._handle_group_coordinator_response, future) @@ -669,6 +673,9 @@ def _send_group_coordinator_request(self): return future def _handle_group_coordinator_response(self, future, response): + if response.API_VERSION >= 1 and response.throttle_time_ms > 0: + log.warning("FindCoordinatorRequest throttled by broker (%d ms)", response.throttle_time_ms) + log.debug("Received group coordinator response %s", response) error_type = Errors.for_code(response.error_code) From a7fdcfd3fe0783c0c9415a06128fb54f36935207 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Wed, 26 Feb 2025 17:29:45 -0800 Subject: [PATCH 5/6] fixup --- kafka/coordinator/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka/coordinator/base.py b/kafka/coordinator/base.py index 354663a54..8f588aa32 100644 --- a/kafka/coordinator/base.py +++ b/kafka/coordinator/base.py @@ -663,9 +663,9 @@ def _send_group_coordinator_request(self): self.group_id, node_id) version = self._client.api_version(FindCoordinatorRequest, max_version=2) if version == 0: - request = FindCoordinatorRequest[0](self.group_id) + request = FindCoordinatorRequest[version](self.group_id) else: - request = FindCoordinatorRequest[0](self.group_id, 0) + request = FindCoordinatorRequest[version](self.group_id, 0) future = Future() _f = self._client.send(node_id, request) _f.add_callback(self._handle_group_coordinator_response, future) From 46e4e4f7d1b64ca0ea8c7ded8c0f07cca24b4995 Mon Sep 17 00:00:00 2001 From: Dana Powers Date: Wed, 26 Feb 2025 17:55:14 -0800 Subject: [PATCH 6/6] admin: drop version check in response handler --- kafka/admin/client.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/kafka/admin/client.py b/kafka/admin/client.py index 10874444c..a46cf9c58 100644 --- a/kafka/admin/client.py +++ b/kafka/admin/client.py @@ -306,18 +306,13 @@ def _find_coordinator_id_process_response(self, response): Returns: The node_id of the broker that is the coordinator. """ - if response.API_VERSION <= 0: - error_type = Errors.for_code(response.error_code) - if error_type is not Errors.NoError: - # Note: When error_type.retriable, Java will retry... see - # KafkaAdminClient's handleFindCoordinatorError method - raise error_type( - "FindCoordinatorRequest failed with response '{}'." - .format(response)) - else: - raise NotImplementedError( - "Support for FindCoordinatorRequest_v{} has not yet been added to KafkaAdminClient." - .format(response.API_VERSION)) + error_type = Errors.for_code(response.error_code) + if error_type is not Errors.NoError: + # Note: When error_type.retriable, Java will retry... see + # KafkaAdminClient's handleFindCoordinatorError method + raise error_type( + "FindCoordinatorRequest failed with response '{}'." + .format(response)) return response.coordinator_id def _find_coordinator_ids(self, group_ids):