diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 59ad718cf..1b0d71c89 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,198 +1,80 @@ -name: CI/CD +# Derived from https://github.com/actions/starter-workflows/blob/main/ci/python-package.yml +# +name: Python Package on: push: branches: ["master"] pull_request: branches: ["master"] - release: - types: [created] - branches: - - 'master' - workflow_dispatch: env: FORCE_COLOR: "1" # Make tools pretty. PIP_DISABLE_PIP_VERSION_CHECK: "1" PIP_NO_PYTHON_VERSION_WARNING: "1" - PYTHON_LATEST: "3.12" - KAFKA_LATEST: "2.6.0" - - # For re-actors/checkout-python-sdist - sdist-artifact: python-package-distributions jobs: + build: - build-sdist: - name: 📦 Build the source distribution - runs-on: ubuntu-latest - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_LATEST }} - cache: pip - - run: python -m pip install build - name: Install core libraries for build and install - - name: Build artifacts - run: python -m build - - name: Upload built artifacts for testing - uses: actions/upload-artifact@v3 - with: - name: ${{ env.sdist-artifact }} - # NOTE: Exact expected file names are specified here - # NOTE: as a safety measure — if anything weird ends - # NOTE: up being in this dir or not all dists will be - # NOTE: produced, this will fail the workflow. - path: dist/${{ env.sdist-name }} - retention-days: 15 - - test-python: - name: Tests on ${{ matrix.python-version }} - needs: - - build-sdist - runs-on: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} - strategy: - fail-fast: false - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" - - "3.12" - experimental: [ false ] - include: - - python-version: "pypy3.9" - experimental: true - - python-version: "~3.13.0-0" - experimental: true - steps: - - name: Checkout the source code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup java - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 11 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: pip - cache-dependency-path: | - requirements-dev.txt - - name: Check Java installation - run: source travis_java_install.sh - - name: Pull Kafka releases - run: ./build_integration.sh - env: - PLATFORM: ${{ matrix.platform }} - KAFKA_VERSION: ${{ env.KAFKA_LATEST }} - # TODO: Cache releases to expedite testing - - name: Install dependencies - run: | - sudo apt install -y libsnappy-dev libzstd-dev - python -m pip install --upgrade pip - python -m pip install tox tox-gh-actions - pip install . - pip install -r requirements-dev.txt - - name: Test with tox - run: tox - env: - PLATFORM: ${{ matrix.platform }} - KAFKA_VERSION: ${{ env.KAFKA_LATEST }} - - test-kafka: - name: Tests for Kafka ${{ matrix.kafka-version }} - needs: - - build-sdist runs-on: ubuntu-latest + name: "Test: python ${{ matrix.python }} / kafka ${{ matrix.kafka }}" + continue-on-error: ${{ matrix.experimental || false }} strategy: fail-fast: false matrix: - kafka-version: + kafka: - "0.8.2.2" - "0.9.0.1" - "0.10.2.2" - - "0.11.0.2" - "0.11.0.3" - "1.1.1" - "2.4.0" - - "2.5.0" - - "2.6.0" + - "2.8.2" + - "3.0.2" + - "3.5.2" + - "3.9.0" + python: + - "3.12" + include: + #- python: "pypy3.9" + # kafka: "2.6.0" + # experimental: true + #- python: "~3.13.0-0" + # kafka: "2.6.0" + # experimental: true + - python: "3.8" + kafka: "3.9.0" + - python: "3.9" + kafka: "3.9.0" + - python: "3.10" + kafka: "3.9.0" + - python: "3.11" + kafka: "3.9.0" + steps: - - name: Checkout the source code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup java - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 8 - - name: Set up Python + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: - python-version: ${{ env.PYTHON_LATEST }} + python-version: ${{ matrix.python }} cache: pip cache-dependency-path: | requirements-dev.txt - - name: Pull Kafka releases - run: ./build_integration.sh - env: - # This is fast enough as long as you pull only one release at a time, - # no need to worry about caching - PLATFORM: ${{ matrix.platform }} - KAFKA_VERSION: ${{ matrix.kafka-version }} - name: Install dependencies run: | sudo apt install -y libsnappy-dev libzstd-dev python -m pip install --upgrade pip - python -m pip install tox tox-gh-actions - pip install . pip install -r requirements-dev.txt - - name: Test with tox - run: tox - env: - PLATFORM: ${{ matrix.platform }} - KAFKA_VERSION: ${{ matrix.kafka-version }} - - check: # This job does nothing and is only used for the branch protection - name: ✅ Ensure the required checks passing - if: always() - needs: - - build-sdist - - test-python - - test-kafka - runs-on: ubuntu-latest - steps: - - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} - publish: - name: 📦 Publish to PyPI - runs-on: ubuntu-latest - needs: [build-sdist] - permissions: - id-token: write - environment: pypi - if: github.event_name == 'release' && github.event.action == 'created' - steps: - - name: Download the sdist artifact - uses: actions/download-artifact@v3 - with: - name: artifact - path: dist - - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Pylint + run: pylint --recursive=y --errors-only --exit-zero kafka test + - name: Setup java + uses: actions/setup-java@v4 with: - password: ${{ secrets.PYPI_API_TOKEN }} + distribution: temurin + java-version: 23 + - name: Pull Kafka release + run: make servers/${{ matrix.kafka }}/kafka-bin + - name: Pytest + run: make test + env: + KAFKA_VERSION: ${{ matrix.kafka }} diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..31dbf0d70 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,35 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # You can also specify other tool versions: + # nodejs: "20" + # rust: "1.70" + # golang: "1.20" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + # Fail on all warnings to avoid broken references + # fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 21e51f5ed..000000000 --- a/.travis.yml +++ /dev/null @@ -1,46 +0,0 @@ -language: python - -dist: xenial - -python: - - 2.7 - - 3.4 - - 3.7 - - 3.8 - - pypy2.7-6.0 - -env: - - KAFKA_VERSION=0.8.2.2 - - KAFKA_VERSION=0.9.0.1 - - KAFKA_VERSION=0.10.2.2 - - KAFKA_VERSION=0.11.0.3 - - KAFKA_VERSION=1.1.1 - - KAFKA_VERSION=2.4.0 - - KAFKA_VERSION=2.5.0 - - KAFKA_VERSION=2.6.0 - -addons: - apt: - packages: - - libsnappy-dev - - libzstd-dev - - openjdk-8-jdk - -cache: - directories: - - $HOME/.cache/pip - - servers/dist - -before_install: - - source travis_java_install.sh - - ./build_integration.sh - -install: - - pip install tox coveralls - - pip install . - -script: - - tox -e `if [ "$TRAVIS_PYTHON_VERSION" == "pypy2.7-6.0" ]; then echo pypy; else echo py${TRAVIS_PYTHON_VERSION/./}; fi` - -after_success: - - coveralls diff --git a/CHANGES.md b/CHANGES.md index 097c55db6..ee28a84e7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,131 @@ +# 2.0.6 (Mar 4, 2025) + +Networking +* Improve error handling in `client._maybe_connect` (#2504) +* Client connection / `maybe_refresh_metadata` changes (#2507) +* Improve too-large timeout handling in client poll +* Default `client.check_version` timeout to `api_version_auto_timeout_ms` (#2496) + +Fixes +* Decode and skip transactional control records in consumer (#2499) +* try / except in consumer coordinator `__del__` + +Testing +* test_conn fixup for py2 + +Project Maintenance +* Add 2.0 branch for backports + +# 2.0.5 (Feb 25, 2025) + +Networking +* Remove unused client bootstrap backoff code +* 200ms timeout for client.poll in ensure_active_group and admin client + +Fixes +* Admin client: check_version only if needed, use node_id kwarg for controller +* Check for -1 controller_id in admin client +* Only acquire coordinator lock in heartbeat thread close if not self thread + +Testing +* Also sleep when waiting for consumers in test_describe_consumer_group_exists +* Refactor sasl_integration test_client - wait for node ready; use send future +* Add timeout to test_kafka_consumer +* Add error str to assert_message_count checks +* Retry on error in test fixture create_topic_via_metadata +* Fixup variable interpolation in test fixture error + +Documentation +* Update compatibility docs +* Include client_id in BrokerConnection __str__ output + +Project Maintenance +* Add make targets `servers/*/api_versions` and `servers/*/messages` + +# 2.0.4 (Feb 21, 2025) + +Networking +* Check for wakeup socket errors on read and close and reinit to reset (#2482) +* Improve client networking backoff / retry (#2480) +* Check for socket and unresolved futures before creating selector in conn.check_version (#2477) +* Handle socket init errors, e.g., when IPv6 is disabled (#2476) + +Fixes +* Avoid self-join in heartbeat thread close (#2488) + +Error Handling +* Always log broker errors in producer.send (#2478) +* Retain unrecognized broker response error codes with dynamic error class (#2481) +* Update kafka.errors with latest types (#2485) + +Compatibility +* Do not validate snappy xerial header version and compat fields (for redpanda) (#2483) + +Documentation +* Added missing docstrings in admin/client.py (#2487) + +Testing +* Update kafka broker test matrix; test against 3.9.0 (#2486) +* Add default resources for new kafka server fixtures (#2484) +* Drop make test-local; add PYTESTS configuration var +* Fix pytest runs when KAFKA_VERSION is not set + +Project Maintenance +* Migrate to pyproject.toml / PEP-621 +* Remove old travis files; update compatibility tests link to gha + +# 2.0.3 (Feb 12, 2025) + +Improvements +* Add optional compression libs to extras_require (#2123, #2387) +* KafkaConsumer: Exit poll if consumer is closed (#2152) +* Support configuration of custom kafka client for Admin/Consumer/Producer (#2144) +* Core Protocol: Add support for flexible versions (#2151) +* (Internal) Allow disabling thread wakeup in _send_request_to_node (#2335) +* Change loglevel of cancelled errors to info (#2467) +* Strip trailing dot off hostname for SSL validation. (#2472) +* Log connection close(error) at ERROR level (#2473) +* Support DescribeLogDirs admin api (#2475) + +Compatibility +* Support for python 3.12 (#2379, #2382) +* Kafka 2.5 / 2.6 (#2162) +* Try collections.abc imports in vendored selectors34 (#2394) +* Catch OSError when checking for gssapi import for windows compatibility (#2407) +* Update vendored six to 1.16.0 (#2398) + +Documentation +* Update usage.rst (#2308, #2334) +* Fix typos (#2319, #2207, #2178) +* Fix links to the compatibility page (#2295, #2226) +* Cleanup install instructions for optional libs (#2139) +* Update license_file to license_files (#2462) +* Update some RST documentation syntax (#2463) +* Add .readthedocs.yaml; update copyright date (#2474) + +Fixes +* Use isinstance in builtin crc32 (#2329) +* Use six.viewitems instead of six.iteritems to avoid encoding problems in StickyPartitionAssignor (#2154) +* Fix array encoding TypeError: object of type 'dict_itemiterator' has no len() (#2167) +* Only try to update sensors fetch lag if the unpacked list contains elements (#2158) +* Avoid logging errors during test fixture cleanup (#2458) +* Release coordinator lock before calling maybe_leave_group (#2460) +* Dont raise RuntimeError for dead process in SpawnedService.wait_for() (#2461) +* Cast the size of a MemoryRecordsBuilder object (#2438) +* Fix DescribeConfigsResponse_v1 config_source (#2464) +* Fix base class of DescribeClientQuotasResponse_v0 (#2465) +* Update socketpair w/ CVE-2024-3219 fix (#2468) + +Testing +* Transition CI/CD to GitHub Workflows (#2378, #2392, #2381, #2406, #2419, #2418, #2417, #2456) +* Refactor Makefile (#2457) +* Use assert_called_with in client_async tests (#2375) +* Cover sticky assignor's metadata method with tests (#2161) +* Update fixtures.py to check "127.0.0.1" for auto port assignment (#2384) +* Use -Djava.security.manager=allow for Java 23 sasl tests (#2469) +* Test with Java 23 (#2470) +* Update kafka properties template; disable group rebalance delay (#2471) + # 2.0.2 (Sep 29, 2020) Consumer diff --git a/Makefile b/Makefile index fc8fa5b21..2df1c6696 100644 --- a/Makefile +++ b/Makefile @@ -1,35 +1,31 @@ -# Some simple testing tasks (sorry, UNIX only). +# Some simple testing tasks -FLAGS= -KAFKA_VERSION=0.11.0.2 -SCALA_VERSION=2.12 +SHELL = bash -setup: - pip install -r requirements-dev.txt - pip install -Ue . +export KAFKA_VERSION ?= 2.4.0 +DIST_BASE_URL ?= https://archive.apache.org/dist/kafka/ -servers/$(KAFKA_VERSION)/kafka-bin: - KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) ./build_integration.sh +# Required to support testing old kafka versions on newer java releases +# The performance opts defaults are set in each kafka brokers bin/kafka_run_class.sh file +# The values here are taken from the 2.4.0 release. +# Note that kafka versions 2.0-2.3 crash on newer java releases; openjdk@11 should work with with "-Djava.security.manager=allow" removed from performance opts +export KAFKA_JVM_PERFORMANCE_OPTS?=-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -Djava.awt.headless=true -Djava.security.manager=allow -build-integration: servers/$(KAFKA_VERSION)/kafka-bin +PYTESTS ?= 'test' -# Test and produce coverage using tox. This is the same as is run on Travis -test37: build-integration - KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) tox -e py37 -- $(FLAGS) +setup: + pip install -r requirements-dev.txt + pip install -Ue . -test27: build-integration - KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) tox -e py27 -- $(FLAGS) +lint: + pylint --recursive=y --errors-only kafka test -# Test using pytest directly if you want to use local python. Useful for other -# platforms that require manual installation for C libraries, ie. Windows. -test-local: build-integration - KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) pytest \ - --pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF $(FLAGS) kafka test +test: build-integration + pytest --durations=10 $(PYTESTS) cov-local: build-integration - KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) pytest \ - --pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF --cov=kafka \ - --cov-config=.covrc --cov-report html $(FLAGS) kafka test + pytest --pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF --cov=kafka \ + --cov-config=.covrc --cov-report html $(TEST_FLAGS) kafka test @echo "open file://`pwd`/htmlcov/index.html" # Check the readme for syntax errors, which can lead to invalid formatting on @@ -56,4 +52,60 @@ doc: make -C docs html @echo "open file://`pwd`/docs/_build/html/index.html" -.PHONY: all test37 test27 test-local cov-local clean doc +.PHONY: all test test-local cov-local clean doc dist publish + +kafka_artifact_version=$(lastword $(subst -, ,$(1))) + +# Mappings for artifacts -> scala version; any unlisted will use default 2.12 +kafka_scala_0_8_0=2.8.0 +kafka_scala_0_8_1=2.10 +kafka_scala_0_8_1_1=2.10 +kafka_scala_0_8_2_0=2.11 +kafka_scala_0_8_2_1=2.11 +kafka_scala_0_8_2_2=2.11 +kafka_scala_0_9_0_0=2.11 +kafka_scala_0_9_0_1=2.11 +kafka_scala_0_10_0_0=2.11 +kafka_scala_0_10_0_1=2.11 +kafka_scala_0_10_1_0=2.11 +scala_version=$(if $(SCALA_VERSION),$(SCALA_VERSION),$(if $(kafka_scala_$(subst .,_,$(1))),$(kafka_scala_$(subst .,_,$(1))),2.12)) + +kafka_artifact_name=kafka_$(call scala_version,$(1))-$(1).$(if $(filter 0.8.0,$(1)),tar.gz,tgz) + +build-integration: servers/$(KAFKA_VERSION)/kafka-bin + +servers/dist: + mkdir -p servers/dist + +servers/dist/kafka_%.tgz servers/dist/kafka_%.tar.gz: + @echo "Downloading $(@F)" + wget -nv -P servers/dist/ -N $(DIST_BASE_URL)$(call kafka_artifact_version,$*)/$(@F) + +servers/dist/jakarta.xml.bind-api-2.3.3.jar: + wget -nv -P servers/dist/ -N https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api/2.3.3/jakarta.xml.bind-api-2.3.3.jar + +# to allow us to derive the prerequisite artifact name from the target name +.SECONDEXPANSION: + +servers/%/kafka-bin: servers/dist/$$(call kafka_artifact_name,$$*) | servers/dist + @echo "Extracting kafka $* binaries from $<" + if [ -d "$@" ]; then rm -rf $@.bak; mv $@ $@.bak; fi + mkdir -p $@ + tar xzvf $< -C $@ --strip-components 1 + if [[ "$*" < "1" ]]; then make servers/patch-libs/$*; fi + +servers/%/api_versions: servers/$$*/kafka-bin + KAFKA_VERSION=$* python -m test.fixtures get_api_versions >$@ + +servers/%/messages: servers/$$*/kafka-bin + cd servers/$*/ && jar xvf kafka-bin/libs/kafka-clients-$*.jar common/message/ + mv servers/$*/common/message/ servers/$*/messages/ + rmdir servers/$*/common + +servers/patch-libs/%: servers/dist/jakarta.xml.bind-api-2.3.3.jar | servers/$$*/kafka-bin + cp $< servers/$*/kafka-bin/libs/ + +servers/download/%: servers/dist/$$(call kafka_artifact_name,$$*) | servers/dist ; + +# Avoid removing any pattern match targets as intermediates (without this, .tgz artifacts are removed by make after extraction) +.SECONDARY: diff --git a/README.rst b/README.rst index 78a92a884..2de04c673 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ Kafka Python client ------------------------ -.. image:: https://img.shields.io/badge/kafka-2.6%2C%202.5%2C%202.4%2C%202.3%2C%202.2%2C%202.1%2C%202.0%2C%201.1%2C%201.0%2C%200.11%2C%200.10%2C%200.9%2C%200.8-brightgreen.svg +.. image:: https://img.shields.io/badge/kafka-3.9--0.8-brightgreen.svg :target: https://kafka-python.readthedocs.io/en/master/compatibility.html .. image:: https://img.shields.io/pypi/pyversions/kafka-python.svg :target: https://pypi.python.org/pypi/kafka-python @@ -32,13 +32,15 @@ check code (perhaps using zookeeper or consul). For older brokers, you can achieve something similar by manually assigning different partitions to each consumer instance with config management tools like chef, ansible, etc. This approach will work fine, though it does not support rebalancing on failures. -See +See https://kafka-python.readthedocs.io/en/master/compatibility.html for more details. Please note that the master branch may contain unreleased features. For release documentation, please see readthedocs and/or python's inline help. ->>> pip install kafka-python +.. code-block:: bash + + $ pip install kafka-python KafkaConsumer @@ -48,42 +50,54 @@ KafkaConsumer is a high-level message consumer, intended to operate as similarly as possible to the official java client. Full support for coordinated consumer groups requires use of kafka brokers that support the Group APIs: kafka v0.9+. -See +See https://kafka-python.readthedocs.io/en/master/apidoc/KafkaConsumer.html for API and configuration details. The consumer iterator returns ConsumerRecords, which are simple namedtuples that expose basic message attributes: topic, partition, offset, key, and value: ->>> from kafka import KafkaConsumer ->>> consumer = KafkaConsumer('my_favorite_topic') ->>> for msg in consumer: -... print (msg) +.. code-block:: python + + from kafka import KafkaConsumer + consumer = KafkaConsumer('my_favorite_topic') + for msg in consumer: + print (msg) + +.. code-block:: python + + # join a consumer group for dynamic partition assignment and offset commits + from kafka import KafkaConsumer + consumer = KafkaConsumer('my_favorite_topic', group_id='my_favorite_group') + for msg in consumer: + print (msg) ->>> # join a consumer group for dynamic partition assignment and offset commits ->>> from kafka import KafkaConsumer ->>> consumer = KafkaConsumer('my_favorite_topic', group_id='my_favorite_group') ->>> for msg in consumer: -... print (msg) +.. code-block:: python ->>> # manually assign the partition list for the consumer ->>> from kafka import TopicPartition ->>> consumer = KafkaConsumer(bootstrap_servers='localhost:1234') ->>> consumer.assign([TopicPartition('foobar', 2)]) ->>> msg = next(consumer) + # manually assign the partition list for the consumer + from kafka import TopicPartition + consumer = KafkaConsumer(bootstrap_servers='localhost:1234') + consumer.assign([TopicPartition('foobar', 2)]) + msg = next(consumer) ->>> # Deserialize msgpack-encoded values ->>> consumer = KafkaConsumer(value_deserializer=msgpack.loads) ->>> consumer.subscribe(['msgpackfoo']) ->>> for msg in consumer: -... assert isinstance(msg.value, dict) +.. code-block:: python ->>> # Access record headers. The returned value is a list of tuples ->>> # with str, bytes for key and value ->>> for msg in consumer: -... print (msg.headers) + # Deserialize msgpack-encoded values + consumer = KafkaConsumer(value_deserializer=msgpack.loads) + consumer.subscribe(['msgpackfoo']) + for msg in consumer: + assert isinstance(msg.value, dict) ->>> # Get consumer metrics ->>> metrics = consumer.metrics() +.. code-block:: python + + # Access record headers. The returned value is a list of tuples + # with str, bytes for key and value + for msg in consumer: + print (msg.headers) + +.. code-block:: python + + # Get consumer metrics + metrics = consumer.metrics() KafkaProducer @@ -91,46 +105,64 @@ KafkaProducer KafkaProducer is a high-level, asynchronous message producer. The class is intended to operate as similarly as possible to the official java client. -See +See https://kafka-python.readthedocs.io/en/master/apidoc/KafkaProducer.html for more details. ->>> from kafka import KafkaProducer ->>> producer = KafkaProducer(bootstrap_servers='localhost:1234') ->>> for _ in range(100): -... producer.send('foobar', b'some_message_bytes') +.. code-block:: python + + from kafka import KafkaProducer + producer = KafkaProducer(bootstrap_servers='localhost:1234') + for _ in range(100): + producer.send('foobar', b'some_message_bytes') + +.. code-block:: python + + # Block until a single message is sent (or timeout) + future = producer.send('foobar', b'another_message') + result = future.get(timeout=60) + +.. code-block:: python + + # Block until all pending messages are at least put on the network + # NOTE: This does not guarantee delivery or success! It is really + # only useful if you configure internal batching using linger_ms + producer.flush() + +.. code-block:: python + + # Use a key for hashed-partitioning + producer.send('foobar', key=b'foo', value=b'bar') + +.. code-block:: python + + # Serialize json messages + import json + producer = KafkaProducer(value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send('fizzbuzz', {'foo': 'bar'}) ->>> # Block until a single message is sent (or timeout) ->>> future = producer.send('foobar', b'another_message') ->>> result = future.get(timeout=60) +.. code-block:: python ->>> # Block until all pending messages are at least put on the network ->>> # NOTE: This does not guarantee delivery or success! It is really ->>> # only useful if you configure internal batching using linger_ms ->>> producer.flush() + # Serialize string keys + producer = KafkaProducer(key_serializer=str.encode) + producer.send('flipflap', key='ping', value=b'1234') ->>> # Use a key for hashed-partitioning ->>> producer.send('foobar', key=b'foo', value=b'bar') +.. code-block:: python ->>> # Serialize json messages ->>> import json ->>> producer = KafkaProducer(value_serializer=lambda v: json.dumps(v).encode('utf-8')) ->>> producer.send('fizzbuzz', {'foo': 'bar'}) + # Compress messages + producer = KafkaProducer(compression_type='gzip') + for i in range(1000): + producer.send('foobar', b'msg %d' % i) ->>> # Serialize string keys ->>> producer = KafkaProducer(key_serializer=str.encode) ->>> producer.send('flipflap', key='ping', value=b'1234') +.. code-block:: python ->>> # Compress messages ->>> producer = KafkaProducer(compression_type='gzip') ->>> for i in range(1000): -... producer.send('foobar', b'msg %d' % i) + # Include record headers. The format is list of tuples with string key + # and bytes value. + producer.send('foobar', value=b'c29tZSB2YWx1ZQ==', headers=[('content-encoding', b'base64')]) ->>> # Include record headers. The format is list of tuples with string key ->>> # and bytes value. ->>> producer.send('foobar', value=b'c29tZSB2YWx1ZQ==', headers=[('content-encoding', b'base64')]) +.. code-block:: python ->>> # Get producer performance metrics ->>> metrics = producer.metrics() + # Get producer performance metrics + metrics = producer.metrics() Thread safety @@ -154,7 +186,7 @@ kafka-python supports the following compression formats: - Zstandard (zstd) gzip is supported natively, the others require installing additional libraries. -See for more information. +See https://kafka-python.readthedocs.io/en/master/install.html for more information. Optimized CRC32 Validation @@ -163,7 +195,7 @@ Optimized CRC32 Validation Kafka uses CRC32 checksums to validate messages. kafka-python includes a pure python implementation for compatibility. To improve performance for high-throughput applications, kafka-python will use `crc32c` for optimized native code if installed. -See for installation instructions. +See https://kafka-python.readthedocs.io/en/master/install.html for installation instructions. See https://pypi.org/project/crc32c/ for details on the underlying crc32c lib. diff --git a/build_integration.sh b/build_integration.sh deleted file mode 100755 index c020b0fe2..000000000 --- a/build_integration.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -: ${ALL_RELEASES:="0.8.2.2 0.9.0.1 0.10.1.1 0.10.2.2 0.11.0.3 1.0.2 1.1.1 2.0.1 2.1.1 2.2.1 2.3.0 2.4.0 2.5.0"} -: ${SCALA_VERSION:=2.11} -: ${DIST_BASE_URL:=https://archive.apache.org/dist/kafka/} -: ${KAFKA_SRC_GIT:=https://github.com/apache/kafka.git} - -# On travis CI, empty KAFKA_VERSION means skip integration tests -# so we don't try to get binaries -# Otherwise it means test all official releases, so we get all of them! -if [ -z "$KAFKA_VERSION" -a -z "$TRAVIS" ]; then - KAFKA_VERSION=$ALL_RELEASES -fi - -pushd servers - mkdir -p dist - pushd dist - for kafka in $KAFKA_VERSION; do - if [ "$kafka" == "trunk" ]; then - if [ ! -d "$kafka" ]; then - git clone $KAFKA_SRC_GIT $kafka - fi - pushd $kafka - git pull - ./gradlew -PscalaVersion=$SCALA_VERSION -Pversion=$kafka releaseTarGz -x signArchives - popd - # Not sure how to construct the .tgz name accurately, so use a wildcard (ugh) - tar xzvf $kafka/core/build/distributions/kafka_*.tgz -C ../$kafka/ - rm $kafka/core/build/distributions/kafka_*.tgz - rm -rf ../$kafka/kafka-bin - mv ../$kafka/kafka_* ../$kafka/kafka-bin - else - echo "-------------------------------------" - echo "Checking kafka binaries for ${kafka}" - echo - if [ "$kafka" == "0.8.0" ]; then - KAFKA_ARTIFACT="kafka_2.8.0-${kafka}.tar.gz" - else if [ "$kafka" \> "2.4.0" ]; then - KAFKA_ARTIFACT="kafka_2.12-${kafka}.tgz" - else - KAFKA_ARTIFACT="kafka_${SCALA_VERSION}-${kafka}.tgz" - fi - fi - if [ ! -f "../$kafka/kafka-bin/bin/kafka-run-class.sh" ]; then - if [ -f "${KAFKA_ARTIFACT}" ]; then - echo "Using cached artifact: ${KAFKA_ARTIFACT}" - else - echo "Downloading kafka ${kafka} tarball" - TARBALL=${DIST_BASE_URL}${kafka}/${KAFKA_ARTIFACT} - if command -v wget 2>/dev/null; then - wget -N $TARBALL - else - echo "wget not found... using curl" - curl -f $TARBALL -o ${KAFKA_ARTIFACT} - fi - fi - echo - echo "Extracting kafka ${kafka} binaries" - tar xzvf ${KAFKA_ARTIFACT} -C ../$kafka/ - rm -rf ../$kafka/kafka-bin - mv ../$kafka/${KAFKA_ARTIFACT/%.t*/} ../$kafka/kafka-bin - if [ ! -f "../$kafka/kafka-bin/bin/kafka-run-class.sh" ]; then - echo "Extraction Failed ($kafka/kafka-bin/bin/kafka-run-class.sh does not exist)!" - exit 1 - fi - else - echo "$kafka is already installed in servers/$kafka/ -- skipping" - fi - fi - echo - done - popd -popd diff --git a/docs/changelog.rst b/docs/changelog.rst index 446b29021..3216ad8ff 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,162 @@ Changelog ========= +2.0.6 (Mar 4, 2025) +################### + +Networking +---------- +* Improve error handling in `client._maybe_connect` (#2504) +* Client connection / `maybe_refresh_metadata` changes (#2507) +* Improve too-large timeout handling in client poll +* Default `client.check_version` timeout to `api_version_auto_timeout_ms` (#2496) + +Fixes +----- +* Decode and skip transactional control records in consumer (#2499) +* try / except in consumer coordinator `__del__` + +Testing +------- +* test_conn fixup for py2 + +Project Maintenance +------------------- +* Add 2.0 branch for backports + + +2.0.5 (Feb 25, 2025) +#################### + +Networking +---------- +* Remove unused client bootstrap backoff code +* 200ms timeout for client.poll in ensure_active_group and admin client + +Fixes +----- +* Admin client: check_version only if needed, use node_id kwarg for controller +* Check for -1 controller_id in admin client +* Only acquire coordinator lock in heartbeat thread close if not self thread + +Testing +------- +* Also sleep when waiting for consumers in test_describe_consumer_group_exists +* Refactor sasl_integration test_client - wait for node ready; use send future +* Add timeout to test_kafka_consumer +* Add error str to assert_message_count checks +* Retry on error in test fixture create_topic_via_metadata +* Fixup variable interpolation in test fixture error + +Documentation +------------- +* Update compatibility docs +* Include client_id in BrokerConnection __str__ output + +Project Maintenance +------------------- +* Add make targets `servers/*/api_versions` and `servers/*/messages` + + +2.0.4 (Feb 21, 2025) +#################### + +Networking +---------- +* Check for wakeup socket errors on read and close and reinit to reset (#2482) +* Improve client networking backoff / retry (#2480) +* Check for socket and unresolved futures before creating selector in conn.check_version (#2477) +* Handle socket init errors, e.g., when IPv6 is disabled (#2476) + +Fixes +----- +* Avoid self-join in heartbeat thread close (#2488) + +Error Handling +-------------- +* Always log broker errors in producer.send (#2478) +* Retain unrecognized broker response error codes with dynamic error class (#2481) +* Update kafka.errors with latest types (#2485) + +Compatibility +------------- +* Do not validate snappy xerial header version and compat fields (for redpanda) (#2483) + +Documentation +------------- +* Added missing docstrings in admin/client.py (#2487) + +Testing +------- +* Update kafka broker test matrix; test against 3.9.0 (#2486) +* Add default resources for new kafka server fixtures (#2484) +* Drop make test-local; add PYTESTS configuration var +* Fix pytest runs when KAFKA_VERSION is not set + +Project Maintenance +------------------- +* Migrate to pyproject.toml / PEP-621 +* Remove old travis files; update compatibility tests link to gha + + +2.0.3 (Feb 12, 2025) +#################### + +Improvements +------------ +* Add optional compression libs to extras_require (#2123, #2387) +* KafkaConsumer: Exit poll if consumer is closed (#2152) +* Support configuration of custom kafka client for Admin/Consumer/Producer (#2144) +* Core Protocol: Add support for flexible versions (#2151) +* (Internal) Allow disabling thread wakeup in _send_request_to_node (#2335) +* Change loglevel of cancelled errors to info (#2467) +* Strip trailing dot off hostname for SSL validation. (#2472) +* Log connection close(error) at ERROR level (#2473) +* Support DescribeLogDirs admin api (#2475) + +Compatibility +------------- +* Support for python 3.12 (#2379, #2382) +* Kafka 2.5 / 2.6 (#2162) +* Try collections.abc imports in vendored selectors34 (#2394) +* Catch OSError when checking for gssapi import for windows compatibility (#2407) +* Update vendored six to 1.16.0 (#2398) + +Documentation +------------- +* Update usage.rst (#2308, #2334) +* Fix typos (#2319, #2207, #2178) +* Fix links to the compatibility page (#2295, #2226) +* Cleanup install instructions for optional libs (#2139) +* Update license_file to license_files (#2462) +* Update some RST documentation syntax (#2463) +* Add .readthedocs.yaml; update copyright date (#2474) + +Fixes +----- +* Use isinstance in builtin crc32 (#2329) +* Use six.viewitems instead of six.iteritems to avoid encoding problems in StickyPartitionAssignor (#2154) +* Fix array encoding TypeError: object of type 'dict_itemiterator' has no len() (#2167) +* Only try to update sensors fetch lag if the unpacked list contains elements (#2158) +* Avoid logging errors during test fixture cleanup (#2458) +* Release coordinator lock before calling maybe_leave_group (#2460) +* Dont raise RuntimeError for dead process in SpawnedService.wait_for() (#2461) +* Cast the size of a MemoryRecordsBuilder object (#2438) +* Fix DescribeConfigsResponse_v1 config_source (#2464) +* Fix base class of DescribeClientQuotasResponse_v0 (#2465) +* Update socketpair w/ CVE-2024-3219 fix (#2468) + +Testing +------- +* Transition CI/CD to GitHub Workflows (#2378, #2392, #2381, #2406, #2419, #2418, #2417, #2456) +* Refactor Makefile (#2457) +* Use assert_called_with in client_async tests (#2375) +* Cover sticky assignor's metadata method with tests (#2161) +* Update fixtures.py to check "127.0.0.1" for auto port assignment (#2384) +* Use -Djava.security.manager=allow for Java 23 sasl tests (#2469) +* Test with Java 23 (#2470) +* Update kafka properties template; disable group rebalance delay (#2471) + 2.0.2 (Sep 29, 2020) #################### @@ -1243,7 +1399,7 @@ Consumers * Improve FailedPayloadsError handling in KafkaConsumer (dpkp PR 398) * KafkaConsumer: avoid raising KeyError in task_done (dpkp PR 389) * MultiProcessConsumer -- support configured partitions list (dpkp PR 380) -* Fix SimpleConsumer leadership change handling (dpkp PR 393) +* Fix SimpleConsumer leadership change handling (dpkp PR 393) * Fix SimpleConsumer connection error handling (reAsOn2010 PR 392) * Improve Consumer handling of 'falsy' partition values (wting PR 342) * Fix _offsets call error in KafkaConsumer (hellais PR 376) @@ -1348,7 +1504,7 @@ Internals * Add test timers via nose-timer plugin; list 10 slowest timings by default (dpkp) * Move fetching last known offset logic to a stand alone function (zever - PR 177) * Improve KafkaConnection and add more tests (dpkp - PR 196) -* Raise TypeError if necessary when encoding strings (mdaniel - PR 204) +* Raise TypeError if necessary when encoding strings (mdaniel - PR 204) * Use Travis-CI to publish tagged releases to pypi (tkuhlman / mumrah) * Use official binary tarballs for integration tests and parallelize travis tests (dpkp - PR 193) * Improve new-topic creation handling (wizzat - PR 174) @@ -1362,7 +1518,7 @@ Internals * Fix connection error timeout and improve tests (wizzat - PR 158) * SimpleProducer randomization of initial round robin ordering (alexcb - PR 139) * Fix connection timeout in KafkaClient and KafkaConnection (maciejkula - PR 161) -* Fix seek + commit behavior (wizzat - PR 148) +* Fix seek + commit behavior (wizzat - PR 148) 0.9.0 (Mar 21, 2014) diff --git a/docs/compatibility.rst b/docs/compatibility.rst index b3ad00634..d9e2ba957 100644 --- a/docs/compatibility.rst +++ b/docs/compatibility.rst @@ -1,12 +1,12 @@ Compatibility ------------- -.. image:: https://img.shields.io/badge/kafka-2.6%2C%202.5%2C%202.4%2C%202.3%2C%202.2%2C%202.1%2C%202.0%2C%201.1%2C%201.0%2C%200.11%2C%200.10%2C%200.9%2C%200.8-brightgreen.svg +.. image:: https://img.shields.io/badge/kafka-3.9--0.8-brightgreen.svg :target: https://kafka-python.readthedocs.io/compatibility.html .. image:: https://img.shields.io/pypi/pyversions/kafka-python.svg :target: https://pypi.python.org/pypi/kafka-python -kafka-python is compatible with (and tested against) broker versions 2.6 +kafka-python is compatible with (and tested against) broker versions 3.9 through 0.8.0 . kafka-python is not compatible with the 0.8.2-beta release. Because the kafka server protocol is backwards compatible, kafka-python is @@ -16,6 +16,6 @@ Although kafka-python is tested and expected to work on recent broker versions, not all features are supported. Specifically, authentication codecs, and transactional producer/consumer support are not fully implemented. PRs welcome! -kafka-python is tested on python 2.7, 3.4, 3.7, 3.8 and pypy2.7. +kafka-python is tested on python 2.7, and 3.8-3.12. -Builds and tests via Travis-CI. See https://travis-ci.org/dpkp/kafka-python +Builds and tests via Github Actions Workflows. See https://github.com/dpkp/kafka-python/actions diff --git a/docs/conf.py b/docs/conf.py index efa8d0807..6273af0ce 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,11 +13,12 @@ # serve to show the default. import os +import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('../')) # -- General configuration ------------------------------------------------ @@ -48,7 +49,7 @@ # General information about the project. project = u'kafka-python' -copyright = u'2016 -- Dana Powers, David Arthur, and Contributors' +copyright = u'2025 -- Dana Powers, David Arthur, and Contributors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -103,7 +104,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/index.rst b/docs/index.rst index 91e5086cc..5dd4f183a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ kafka-python ############ -.. image:: https://img.shields.io/badge/kafka-2.6%2C%202.5%2C%202.4%2C%202.3%2C%202.2%2C%202.1%2C%202.0%2C%201.1%2C%201.0%2C%200.11%2C%200.10%2C%200.9%2C%200.8-brightgreen.svg +.. image:: https://img.shields.io/badge/kafka-3.9--0.8-brightgreen.svg :target: https://kafka-python.readthedocs.io/en/master/compatibility.html .. image:: https://img.shields.io/pypi/pyversions/kafka-python.svg :target: https://pypi.python.org/pypi/kafka-python @@ -31,7 +31,9 @@ failures. See `Compatibility `_ for more details. Please note that the master branch may contain unreleased features. For release documentation, please see readthedocs and/or python's inline help. ->>> pip install kafka-python +.. code:: bash + + pip install kafka-python KafkaConsumer @@ -47,28 +49,36 @@ See `KafkaConsumer `_ for API and configuration detai The consumer iterator returns ConsumerRecords, which are simple namedtuples that expose basic message attributes: topic, partition, offset, key, and value: ->>> from kafka import KafkaConsumer ->>> consumer = KafkaConsumer('my_favorite_topic') ->>> for msg in consumer: -... print (msg) +.. code:: python + + from kafka import KafkaConsumer + consumer = KafkaConsumer('my_favorite_topic') + for msg in consumer: + print (msg) + +.. code:: python ->>> # join a consumer group for dynamic partition assignment and offset commits ->>> from kafka import KafkaConsumer ->>> consumer = KafkaConsumer('my_favorite_topic', group_id='my_favorite_group') ->>> for msg in consumer: -... print (msg) + # join a consumer group for dynamic partition assignment and offset commits + from kafka import KafkaConsumer + consumer = KafkaConsumer('my_favorite_topic', group_id='my_favorite_group') + for msg in consumer: + print (msg) ->>> # manually assign the partition list for the consumer ->>> from kafka import TopicPartition ->>> consumer = KafkaConsumer(bootstrap_servers='localhost:1234') ->>> consumer.assign([TopicPartition('foobar', 2)]) ->>> msg = next(consumer) +.. code:: python ->>> # Deserialize msgpack-encoded values ->>> consumer = KafkaConsumer(value_deserializer=msgpack.loads) ->>> consumer.subscribe(['msgpackfoo']) ->>> for msg in consumer: -... assert isinstance(msg.value, dict) + # manually assign the partition list for the consumer + from kafka import TopicPartition + consumer = KafkaConsumer(bootstrap_servers='localhost:1234') + consumer.assign([TopicPartition('foobar', 2)]) + msg = next(consumer) + +.. code:: python + + # Deserialize msgpack-encoded values + consumer = KafkaConsumer(value_deserializer=msgpack.loads) + consumer.subscribe(['msgpackfoo']) + for msg in consumer: + assert isinstance(msg.value, dict) KafkaProducer @@ -78,36 +88,50 @@ KafkaProducer The class is intended to operate as similarly as possible to the official java client. See `KafkaProducer `_ for more details. ->>> from kafka import KafkaProducer ->>> producer = KafkaProducer(bootstrap_servers='localhost:1234') ->>> for _ in range(100): -... producer.send('foobar', b'some_message_bytes') +.. code:: python + + from kafka import KafkaProducer + producer = KafkaProducer(bootstrap_servers='localhost:1234') + for _ in range(100): + producer.send('foobar', b'some_message_bytes') + +.. code:: python + + # Block until a single message is sent (or timeout) + future = producer.send('foobar', b'another_message') + result = future.get(timeout=60) + +.. code:: python + + # Block until all pending messages are at least put on the network + # NOTE: This does not guarantee delivery or success! It is really + # only useful if you configure internal batching using linger_ms + producer.flush() + +.. code:: python + + # Use a key for hashed-partitioning + producer.send('foobar', key=b'foo', value=b'bar') ->>> # Block until a single message is sent (or timeout) ->>> future = producer.send('foobar', b'another_message') ->>> result = future.get(timeout=60) +.. code:: python ->>> # Block until all pending messages are at least put on the network ->>> # NOTE: This does not guarantee delivery or success! It is really ->>> # only useful if you configure internal batching using linger_ms ->>> producer.flush() + # Serialize json messages + import json + producer = KafkaProducer(value_serializer=lambda v: json.dumps(v).encode('utf-8')) + producer.send('fizzbuzz', {'foo': 'bar'}) ->>> # Use a key for hashed-partitioning ->>> producer.send('foobar', key=b'foo', value=b'bar') +.. code:: python ->>> # Serialize json messages ->>> import json ->>> producer = KafkaProducer(value_serializer=lambda v: json.dumps(v).encode('utf-8')) ->>> producer.send('fizzbuzz', {'foo': 'bar'}) + # Serialize string keys + producer = KafkaProducer(key_serializer=str.encode) + producer.send('flipflap', key='ping', value=b'1234') ->>> # Serialize string keys ->>> producer = KafkaProducer(key_serializer=str.encode) ->>> producer.send('flipflap', key='ping', value=b'1234') +.. code:: python ->>> # Compress messages ->>> producer = KafkaProducer(compression_type='gzip') ->>> for i in range(1000): -... producer.send('foobar', b'msg %d' % i) + # Compress messages + producer = KafkaProducer(compression_type='gzip') + for i in range(1000): + producer.send('foobar', b'msg %d' % i) Thread safety diff --git a/docs/license.rst b/docs/license.rst index e9d5c9adb..f419915bd 100644 --- a/docs/license.rst +++ b/docs/license.rst @@ -6,5 +6,5 @@ License Apache License, v2.0. See `LICENSE `_. -Copyright 2016, Dana Powers, David Arthur, and Contributors +Copyright 2025, Dana Powers, David Arthur, and Contributors (See `AUTHORS `_). diff --git a/docs/requirements.txt b/docs/requirements.txt index 0f095e074..61a675cab 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ -sphinx -sphinx_rtd_theme +sphinx==8.1.3 +sphinx_rtd_theme==3.0.2 # Install kafka-python in editable mode # This allows the sphinx autodoc module diff --git a/docs/tests.rst b/docs/tests.rst index 561179ca5..988afca65 100644 --- a/docs/tests.rst +++ b/docs/tests.rst @@ -6,12 +6,14 @@ Tests .. image:: https://travis-ci.org/dpkp/kafka-python.svg?branch=master :target: https://travis-ci.org/dpkp/kafka-python -Test environments are managed via tox. The test suite is run via pytest. +The test suite is run via pytest. -Linting is run via pylint, but is generally skipped on pypy due to pylint -compatibility / performance issues. +Linting is run via pylint, but is currently skipped during CI/CD due to +accumulated debt. We'd like to transition to ruff! For test coverage details, see https://coveralls.io/github/dpkp/kafka-python +Coverage reporting is currently disabled as we have transitioned from travis +to GH Actions and have not yet re-enabled coveralls integration. The test suite includes unit tests that mock network interfaces, as well as integration tests that setup and teardown kafka broker (and zookeeper) @@ -21,30 +23,21 @@ fixtures for client / consumer / producer testing. Unit tests ------------------ -To run the tests locally, install tox: +To run the tests locally, install test dependencies: .. code:: bash - pip install tox + pip install -r requirements-dev.txt -For more details, see https://tox.readthedocs.io/en/latest/install.html - -Then simply run tox, optionally setting the python environment. -If unset, tox will loop through all environments. +Then simply run pytest (or make test) from your preferred python + virtualenv. .. code:: bash - tox -e py27 - tox -e py35 - - # run protocol tests only - tox -- -v test.test_protocol - - # re-run the last failing test, dropping into pdb - tox -e py27 -- --lf --pdb + # run protocol tests only (via pytest) + pytest test/test_protocol.py - # see available (pytest) options - tox -e py27 -- --help + # Run conn tests only (via make) + PYTESTS=test/test_conn.py make test Integration tests @@ -52,35 +45,8 @@ Integration tests .. code:: bash - KAFKA_VERSION=0.8.2.2 tox -e py27 - KAFKA_VERSION=1.0.1 tox -e py36 - - -Integration tests start Kafka and Zookeeper fixtures. This requires downloading -kafka server binaries: - -.. code:: bash - - ./build_integration.sh - -By default, this will install the broker versions listed in build_integration.sh's `ALL_RELEASES` -into the servers/ directory. To install a specific version, set the `KAFKA_VERSION` variable: - -.. code:: bash - - KAFKA_VERSION=1.0.1 ./build_integration.sh + KAFKA_VERSION=3.9.0 make test -Then to run the tests against a specific Kafka version, simply set the `KAFKA_VERSION` -env variable to the server build you want to use for testing: - -.. code:: bash - - KAFKA_VERSION=1.0.1 tox -e py36 - -To test against the kafka source tree, set KAFKA_VERSION=trunk -[optionally set SCALA_VERSION (defaults to the value set in `build_integration.sh`)] - -.. code:: bash - SCALA_VERSION=2.12 KAFKA_VERSION=trunk ./build_integration.sh - KAFKA_VERSION=trunk tox -e py36 +Integration tests start Kafka and Zookeeper fixtures. Make will download +kafka server binaries automatically if needed. diff --git a/docs/usage.rst b/docs/usage.rst index 047bbad77..c001ec049 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -28,7 +28,7 @@ KafkaConsumer # consume json messages KafkaConsumer(value_deserializer=lambda m: json.loads(m.decode('ascii'))) - # consume msgpack + # consume msgpack KafkaConsumer(value_deserializer=msgpack.unpackb) # StopIteration if no message after 1sec @@ -104,7 +104,7 @@ KafkaProducer log.error('I am an errback', exc_info=excp) # handle exception - # produce asynchronously with callbacks + # produce asynchronously with callbacks producer.send('my-topic', b'raw_bytes').add_callback(on_send_success).add_errback(on_send_error) # block until all async messages are sent @@ -112,8 +112,8 @@ KafkaProducer # configure multiple retries producer = KafkaProducer(retries=5) - - + + ClusterMetadata ============= .. code:: python @@ -131,7 +131,7 @@ ClusterMetadata # get all partitions of a topic print(clusterMetadata.partitions_for_topic("topic")) - # list topics + # list topics print(clusterMetadata.topics()) @@ -140,9 +140,9 @@ KafkaAdminClient .. code:: python from kafka import KafkaAdminClient from kafka.admin import NewTopic - + admin = KafkaAdminClient(bootstrap_servers=['broker1:1234']) - + # create a new topic topics_list = [] topics_list.append(NewTopic(name="testtopic", num_partitions=1, replication_factor=1)) @@ -160,4 +160,4 @@ KafkaAdminClient # get consumer group offset print(admin.list_consumer_group_offsets('cft-plt-qa.connect')) - + diff --git a/kafka/__init__.py b/kafka/__init__.py index d5e30affa..41a014072 100644 --- a/kafka/__init__.py +++ b/kafka/__init__.py @@ -4,7 +4,7 @@ from kafka.version import __version__ __author__ = 'Dana Powers' __license__ = 'Apache License 2.0' -__copyright__ = 'Copyright 2016 Dana Powers, David Arthur, and Contributors' +__copyright__ = 'Copyright 2025 Dana Powers, David Arthur, and Contributors' # Set default logging handler to avoid "No handler found" warnings. import logging diff --git a/kafka/admin/client.py b/kafka/admin/client.py index 8eb7504a7..27ad69312 100644 --- a/kafka/admin/client.py +++ b/kafka/admin/client.py @@ -1,9 +1,10 @@ -from __future__ import absolute_import +from __future__ import absolute_import, division from collections import defaultdict import copy import logging import socket +import time from . import ConfigResourceType from kafka.vendor import six @@ -20,9 +21,10 @@ from kafka.protocol.admin import ( CreateTopicsRequest, DeleteTopicsRequest, DescribeConfigsRequest, AlterConfigsRequest, CreatePartitionsRequest, ListGroupsRequest, DescribeGroupsRequest, DescribeAclsRequest, CreateAclsRequest, DeleteAclsRequest, - DeleteGroupsRequest + DeleteGroupsRequest, DescribeLogDirsRequest ) -from kafka.protocol.commit import GroupCoordinatorRequest, OffsetFetchRequest +from kafka.protocol.commit import OffsetFetchRequest +from kafka.protocol.find_coordinator import FindCoordinatorRequest from kafka.protocol.metadata import MetadataRequest from kafka.protocol.types import Array from kafka.structs import TopicPartition, OffsetAndMetadata, MemberInformation, GroupInformation @@ -72,7 +74,7 @@ class KafkaAdminClient(object): reconnection attempts will continue periodically with this fixed rate. To avoid connection storms, a randomization factor of 0.2 will be applied to the backoff resulting in a random range between - 20% below and 20% above the computed value. Default: 1000. + 20% below and 20% above the computed value. Default: 30000. request_timeout_ms (int): Client request timeout in milliseconds. Default: 30000. connections_max_idle_ms: Close idle connections after the number of @@ -140,6 +142,9 @@ class KafkaAdminClient(object): Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms. sasl_plain_password (str): password for sasl PLAIN and SCRAM authentication. Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms. + sasl_kerberos_name (str or gssapi.Name): Constructed gssapi.Name for use with + sasl mechanism handshake. If provided, sasl_kerberos_service_name and + sasl_kerberos_domain name are ignored. Default: None. sasl_kerberos_service_name (str): Service name to include in GSSAPI sasl mechanism handshake. Default: 'kafka' sasl_kerberos_domain_name (str): kerberos domain name to use in GSSAPI @@ -156,7 +161,7 @@ class KafkaAdminClient(object): 'request_timeout_ms': 30000, 'connections_max_idle_ms': 9 * 60 * 1000, 'reconnect_backoff_ms': 50, - 'reconnect_backoff_max_ms': 1000, + 'reconnect_backoff_max_ms': 30000, 'max_in_flight_requests_per_connection': 5, 'receive_buffer_bytes': None, 'send_buffer_bytes': None, @@ -179,6 +184,7 @@ class KafkaAdminClient(object): 'sasl_mechanism': None, 'sasl_plain_username': None, 'sasl_plain_password': None, + 'sasl_kerberos_name': None, 'sasl_kerberos_service_name': 'kafka', 'sasl_kerberos_domain_name': None, 'sasl_oauth_token_provider': None, @@ -212,11 +218,9 @@ def __init__(self, **configs): metric_group_prefix='admin', **self.config ) - self._client.check_version(timeout=(self.config['api_version_auto_timeout_ms'] / 1000)) # Get auto-discovered version from client if necessary - if self.config['api_version'] is None: - self.config['api_version'] = self._client.config['api_version'] + self.config['api_version'] = self._client.config['api_version'] self._closed = False self._refresh_controller_id() @@ -233,58 +237,44 @@ def close(self): self._closed = True log.debug("KafkaAdminClient is now closed.") - def _matching_api_version(self, operation): - """Find the latest version of the protocol operation supported by both - this library and the broker. - - This resolves to the lesser of either the latest api version this - library supports, or the max version supported by the broker. - - :param operation: A list of protocol operation versions from kafka.protocol. - :return: The max matching version number between client and broker. - """ - broker_api_versions = self._client.get_api_versions() - api_key = operation[0].API_KEY - if broker_api_versions is None or api_key not in broker_api_versions: - raise IncompatibleBrokerVersion( - "Kafka broker does not support the '{}' Kafka protocol." - .format(operation[0].__name__)) - min_version, max_version = broker_api_versions[api_key] - version = min(len(operation) - 1, max_version) - if version < min_version: - # max library version is less than min broker version. Currently, - # no Kafka versions specify a min msg version. Maybe in the future? - raise IncompatibleBrokerVersion( - "No version of the '{}' Kafka protocol is supported by both the client and broker." - .format(operation[0].__name__)) - return version - def _validate_timeout(self, timeout_ms): """Validate the timeout is set or use the configuration default. - :param timeout_ms: The timeout provided by api call, in milliseconds. - :return: The timeout to use for the operation. + Arguments: + timeout_ms: The timeout provided by api call, in milliseconds. + + Returns: + The timeout to use for the operation. """ return timeout_ms or self.config['request_timeout_ms'] - def _refresh_controller_id(self): + def _refresh_controller_id(self, timeout_ms=30000): """Determine the Kafka cluster controller.""" - version = self._matching_api_version(MetadataRequest) + version = self._client.api_version(MetadataRequest, max_version=6) if 1 <= version <= 6: - request = MetadataRequest[version]() - future = self._send_request_to_node(self._client.least_loaded_node(), request) - - self._wait_for_futures([future]) - - response = future.value - controller_id = response.controller_id - # verify the controller is new enough to support our requests - controller_version = self._client.check_version(controller_id, timeout=(self.config['api_version_auto_timeout_ms'] / 1000)) - if controller_version < (0, 10, 0): - raise IncompatibleBrokerVersion( - "The controller appears to be running Kafka {}. KafkaAdminClient requires brokers >= 0.10.0.0." - .format(controller_version)) - self._controller_id = controller_id + timeout_at = time.time() + timeout_ms / 1000 + while time.time() < timeout_at: + request = MetadataRequest[version]() + future = self._send_request_to_node(self._client.least_loaded_node(), request) + + self._wait_for_futures([future]) + + response = future.value + controller_id = response.controller_id + if controller_id == -1: + log.warning("Controller ID not available, got -1") + time.sleep(1) + continue + # verify the controller is new enough to support our requests + controller_version = self._client.check_version(node_id=controller_id) + if controller_version < (0, 10, 0): + raise IncompatibleBrokerVersion( + "The controller appears to be running Kafka {}. KafkaAdminClient requires brokers >= 0.10.0.0." + .format(controller_version)) + self._controller_id = controller_id + return + else: + raise Errors.NodeNotAvailableError('controller') else: raise UnrecognizedBrokerVersion( "Kafka Admin interface cannot determine the controller using MetadataRequest_v{}." @@ -293,43 +283,40 @@ def _refresh_controller_id(self): def _find_coordinator_id_send_request(self, group_id): """Send a FindCoordinatorRequest to a broker. - :param group_id: The consumer group ID. This is typically the group + Arguments: + group_id: The consumer group ID. This is typically the group name as a string. - :return: A message future + + Returns: + A message future """ - # TODO add support for dynamically picking version of - # GroupCoordinatorRequest which was renamed to FindCoordinatorRequest. - # When I experimented with this, the coordinator value returned in - # GroupCoordinatorResponse_v1 didn't match the value returned by - # GroupCoordinatorResponse_v0 and I couldn't figure out why. - version = 0 - # version = self._matching_api_version(GroupCoordinatorRequest) + version = self._client.api_version(FindCoordinatorRequest, max_version=2) if version <= 0: - request = GroupCoordinatorRequest[version](group_id) + request = FindCoordinatorRequest[version](group_id) + elif version <= 2: + request = FindCoordinatorRequest[version](group_id, 0) else: raise NotImplementedError( - "Support for GroupCoordinatorRequest_v{} has not yet been added to KafkaAdminClient." + "Support for FindCoordinatorRequest_v{} has not yet been added to KafkaAdminClient." .format(version)) return self._send_request_to_node(self._client.least_loaded_node(), request) def _find_coordinator_id_process_response(self, response): """Process a FindCoordinatorResponse. - :param response: a FindCoordinatorResponse. - :return: The node_id of the broker that is the coordinator. + Arguments: + response: a FindCoordinatorResponse. + + Returns: + The node_id of the broker that is the coordinator. """ - if response.API_VERSION <= 0: - error_type = Errors.for_code(response.error_code) - if error_type is not Errors.NoError: - # Note: When error_type.retriable, Java will retry... see - # KafkaAdminClient's handleFindCoordinatorError method - raise error_type( - "FindCoordinatorRequest failed with response '{}'." - .format(response)) - else: - raise NotImplementedError( - "Support for FindCoordinatorRequest_v{} has not yet been added to KafkaAdminClient." - .format(response.API_VERSION)) + error_type = Errors.for_code(response.error_code) + if error_type is not Errors.NoError: + # Note: When error_type.retriable, Java will retry... see + # KafkaAdminClient's handleFindCoordinatorError method + raise error_type( + "FindCoordinatorRequest failed with response '{}'." + .format(response)) return response.coordinator_id def _find_coordinator_ids(self, group_ids): @@ -339,9 +326,12 @@ def _find_coordinator_ids(self, group_ids): Will block until the FindCoordinatorResponse is received for all groups. Any errors are immediately raised. - :param group_ids: A list of consumer group IDs. This is typically the group + Arguments: + group_ids: A list of consumer group IDs. This is typically the group name as a string. - :return: A dict of {group_id: node_id} where node_id is the id of the + + Returns: + A dict of {group_id: node_id} where node_id is the id of the broker that is the coordinator for the corresponding group. """ groups_futures = { @@ -358,18 +348,24 @@ def _find_coordinator_ids(self, group_ids): def _send_request_to_node(self, node_id, request, wakeup=True): """Send a Kafka protocol message to a specific broker. - Returns a future that may be polled for status and results. + Arguments: + node_id: The broker id to which to send the message. + request: The message to send. - :param node_id: The broker id to which to send the message. - :param request: The message to send. - :param wakeup: Optional flag to disable thread-wakeup. - :return: A future object that may be polled for status and results. - :exception: The exception if the message could not be sent. + + Keyword Arguments: + wakeup (bool, optional): Optional flag to disable thread-wakeup. + + Returns: + A future object that may be polled for status and results. + + Raises: + The exception if the message could not be sent. """ while not self._client.ready(node_id): # poll until the connection to broker is ready, otherwise send() # will fail with NodeNotReadyError - self._client.poll() + self._client.poll(timeout_ms=200) return self._client.send(node_id, request, wakeup) def _send_request_to_controller(self, request): @@ -377,8 +373,11 @@ def _send_request_to_controller(self, request): Will block until the message result is received. - :param request: The message to send. - :return: The Kafka protocol response for the message. + Arguments: + request: The message to send. + + Returns: + The Kafka protocol response for the message. """ tries = 2 # in case our cached self._controller_id is outdated while tries: @@ -418,6 +417,18 @@ def _send_request_to_controller(self, request): @staticmethod def _convert_new_topic_request(new_topic): + """ + Build the tuple required by CreateTopicsRequest from a NewTopic object. + + Arguments: + new_topic: A NewTopic instance containing name, partition count, replication factor, + replica assignments, and config entries. + + Returns: + A tuple in the form: + (topic_name, num_partitions, replication_factor, [(partition_id, [replicas])...], + [(config_key, config_value)...]) + """ return ( new_topic.name, new_topic.num_partitions, @@ -433,14 +444,19 @@ def _convert_new_topic_request(new_topic): def create_topics(self, new_topics, timeout_ms=None, validate_only=False): """Create new topics in the cluster. - :param new_topics: A list of NewTopic objects. - :param timeout_ms: Milliseconds to wait for new topics to be created - before the broker returns. - :param validate_only: If True, don't actually create new topics. - Not supported by all versions. Default: False - :return: Appropriate version of CreateTopicResponse class. + Arguments: + new_topics: A list of NewTopic objects. + + Keyword Arguments: + timeout_ms (numeric, optional): Milliseconds to wait for new topics to be created + before the broker returns. + validate_only (bool, optional): If True, don't actually create new topics. + Not supported by all versions. Default: False + + Returns: + Appropriate version of CreateTopicResponse class. """ - version = self._matching_api_version(CreateTopicsRequest) + version = self._client.api_version(CreateTopicsRequest, max_version=3) timeout_ms = self._validate_timeout(timeout_ms) if version == 0: if validate_only: @@ -468,12 +484,17 @@ def create_topics(self, new_topics, timeout_ms=None, validate_only=False): def delete_topics(self, topics, timeout_ms=None): """Delete topics from the cluster. - :param topics: A list of topic name strings. - :param timeout_ms: Milliseconds to wait for topics to be deleted - before the broker returns. - :return: Appropriate version of DeleteTopicsResponse class. + Arguments: + topics ([str]): A list of topic name strings. + + Keyword Arguments: + timeout_ms (numeric, optional): Milliseconds to wait for topics to be deleted + before the broker returns. + + Returns: + Appropriate version of DeleteTopicsResponse class. """ - version = self._matching_api_version(DeleteTopicsRequest) + version = self._client.api_version(DeleteTopicsRequest, max_version=3) timeout_ms = self._validate_timeout(timeout_ms) if version <= 3: request = DeleteTopicsRequest[version]( @@ -492,7 +513,7 @@ def _get_cluster_metadata(self, topics=None, auto_topic_creation=False): """ topics == None means "get all topics" """ - version = self._matching_api_version(MetadataRequest) + version = self._client.api_version(MetadataRequest, max_version=5) if version <= 3: if auto_topic_creation: raise IncompatibleBrokerVersion( @@ -515,16 +536,38 @@ def _get_cluster_metadata(self, topics=None, auto_topic_creation=False): return future.value def list_topics(self): + """Retrieve a list of all topic names in the cluster. + + Returns: + A list of topic name strings. + """ metadata = self._get_cluster_metadata(topics=None) obj = metadata.to_object() return [t['topic'] for t in obj['topics']] def describe_topics(self, topics=None): + """Fetch metadata for the specified topics or all topics if None. + + Keyword Arguments: + topics ([str], optional) A list of topic names. If None, metadata for all + topics is retrieved. + + Returns: + A list of dicts describing each topic (including partition info). + """ metadata = self._get_cluster_metadata(topics=topics) obj = metadata.to_object() return obj['topics'] def describe_cluster(self): + """ + Fetch cluster-wide metadata such as the list of brokers, the controller ID, + and the cluster ID. + + + Returns: + A dict with cluster-wide metadata, excluding topic details. + """ metadata = self._get_cluster_metadata() obj = metadata.to_object() obj.pop('topics') # We have 'describe_topics' for this @@ -532,6 +575,15 @@ def describe_cluster(self): @staticmethod def _convert_describe_acls_response_to_acls(describe_response): + """Convert a DescribeAclsResponse into a list of ACL objects and a KafkaError. + + Arguments: + describe_response: The response object from the DescribeAclsRequest. + + Returns: + A tuple of (list_of_acl_objects, error) where error is an instance + of KafkaError (NoError if successful). + """ version = describe_response.API_VERSION error = Errors.for_code(describe_response.error_code) @@ -571,11 +623,14 @@ def describe_acls(self, acl_filter): The cluster must be configured with an authorizer for this to work, or you will get a SecurityDisabledError - :param acl_filter: an ACLFilter object - :return: tuple of a list of matching ACL objects and a KafkaError (NoError if successful) + Arguments: + acl_filter: an ACLFilter object + + Returns: + tuple of a list of matching ACL objects and a KafkaError (NoError if successful) """ - version = self._matching_api_version(DescribeAclsRequest) + version = self._client.api_version(DescribeAclsRequest, max_version=1) if version == 0: request = DescribeAclsRequest[version]( resource_type=acl_filter.resource_pattern.resource_type, @@ -617,6 +672,14 @@ def describe_acls(self, acl_filter): @staticmethod def _convert_create_acls_resource_request_v0(acl): + """Convert an ACL object into the CreateAclsRequest v0 format. + + Arguments: + acl: An ACL object with resource pattern and permissions. + + Returns: + A tuple: (resource_type, resource_name, principal, host, operation, permission_type). + """ return ( acl.resource_pattern.resource_type, @@ -629,7 +692,14 @@ def _convert_create_acls_resource_request_v0(acl): @staticmethod def _convert_create_acls_resource_request_v1(acl): + """Convert an ACL object into the CreateAclsRequest v1 format. + Arguments: + acl: An ACL object with resource pattern and permissions. + + Returns: + A tuple: (resource_type, resource_name, pattern_type, principal, host, operation, permission_type). + """ return ( acl.resource_pattern.resource_type, acl.resource_pattern.resource_name, @@ -642,6 +712,19 @@ def _convert_create_acls_resource_request_v1(acl): @staticmethod def _convert_create_acls_response_to_acls(acls, create_response): + """Parse CreateAclsResponse and correlate success/failure with original ACL objects. + + Arguments: + acls: A list of ACL objects that were requested for creation. + create_response: The broker's CreateAclsResponse object. + + Returns: + A dict with: + { + 'succeeded': [list of ACL objects successfully created], + 'failed': [(acl_object, KafkaError), ...] + } + """ version = create_response.API_VERSION creations_error = [] @@ -670,15 +753,18 @@ def create_acls(self, acls): This endpoint only accepts a list of concrete ACL objects, no ACLFilters. Throws TopicAlreadyExistsError if topic is already present. - :param acls: a list of ACL objects - :return: dict of successes and failures + Arguments: + acls: a list of ACL objects + + Returns: + dict of successes and failures """ for acl in acls: if not isinstance(acl, ACL): raise IllegalArgumentError("acls must contain ACL objects") - version = self._matching_api_version(CreateAclsRequest) + version = self._client.api_version(CreateAclsRequest, max_version=1) if version == 0: request = CreateAclsRequest[version]( creations=[self._convert_create_acls_resource_request_v0(acl) for acl in acls] @@ -701,6 +787,14 @@ def create_acls(self, acls): @staticmethod def _convert_delete_acls_resource_request_v0(acl): + """Convert an ACLFilter object into the DeleteAclsRequest v0 format. + + Arguments: + acl: An ACLFilter object identifying the ACLs to be deleted. + + Returns: + A tuple: (resource_type, resource_name, principal, host, operation, permission_type). + """ return ( acl.resource_pattern.resource_type, acl.resource_pattern.resource_name, @@ -712,6 +806,14 @@ def _convert_delete_acls_resource_request_v0(acl): @staticmethod def _convert_delete_acls_resource_request_v1(acl): + """Convert an ACLFilter object into the DeleteAclsRequest v1 format. + + Arguments: + acl: An ACLFilter object identifying the ACLs to be deleted. + + Returns: + A tuple: (resource_type, resource_name, pattern_type, principal, host, operation, permission_type). + """ return ( acl.resource_pattern.resource_type, acl.resource_pattern.resource_name, @@ -724,6 +826,16 @@ def _convert_delete_acls_resource_request_v1(acl): @staticmethod def _convert_delete_acls_response_to_matching_acls(acl_filters, delete_response): + """Parse the DeleteAclsResponse and map the results back to each input ACLFilter. + + Arguments: + acl_filters: A list of ACLFilter objects that were provided in the request. + delete_response: The response from the DeleteAclsRequest. + + Returns: + A list of tuples of the form: + (acl_filter, [(matching_acl, KafkaError), ...], filter_level_error). + """ version = delete_response.API_VERSION filter_result_list = [] for i, filter_responses in enumerate(delete_response.filter_responses): @@ -762,8 +874,11 @@ def delete_acls(self, acl_filters): Deletes all ACLs matching the list of input ACLFilter - :param acl_filters: a list of ACLFilter - :return: a list of 3-tuples corresponding to the list of input filters. + Arguments: + acl_filters: a list of ACLFilter + + Returns: + a list of 3-tuples corresponding to the list of input filters. The tuples hold (the input ACLFilter, list of affected ACLs, KafkaError instance) """ @@ -771,7 +886,7 @@ def delete_acls(self, acl_filters): if not isinstance(acl, ACLFilter): raise IllegalArgumentError("acl_filters must contain ACLFilter type objects") - version = self._matching_api_version(DeleteAclsRequest) + version = self._client.api_version(DeleteAclsRequest, max_version=1) if version == 0: request = DeleteAclsRequest[version]( @@ -795,6 +910,14 @@ def delete_acls(self, acl_filters): @staticmethod def _convert_describe_config_resource_request(config_resource): + """Convert a ConfigResource into the format required by DescribeConfigsRequest. + + Arguments: + config_resource: A ConfigResource with resource_type, name, and optional config keys. + + Returns: + A tuple: (resource_type, resource_name, [list_of_config_keys] or None). + """ return ( config_resource.resource_type, config_resource.name, @@ -806,13 +929,18 @@ def _convert_describe_config_resource_request(config_resource): def describe_configs(self, config_resources, include_synonyms=False): """Fetch configuration parameters for one or more Kafka resources. - :param config_resources: An list of ConfigResource objects. - Any keys in ConfigResource.configs dict will be used to filter the - result. Setting the configs dict to None will get all values. An - empty dict will get zero values (as per Kafka protocol). - :param include_synonyms: If True, return synonyms in response. Not - supported by all versions. Default: False. - :return: Appropriate version of DescribeConfigsResponse class. + Arguments: + config_resources: An list of ConfigResource objects. + Any keys in ConfigResource.configs dict will be used to filter the + result. Setting the configs dict to None will get all values. An + empty dict will get zero values (as per Kafka protocol). + + Keyword Arguments: + include_synonyms (bool, optional): If True, return synonyms in response. Not + supported by all versions. Default: False. + + Returns: + Appropriate version of DescribeConfigsResponse class. """ # Break up requests by type - a broker config request must be sent to the specific broker. @@ -827,7 +955,7 @@ def describe_configs(self, config_resources, include_synonyms=False): topic_resources.append(self._convert_describe_config_resource_request(config_resource)) futures = [] - version = self._matching_api_version(DescribeConfigsRequest) + version = self._client.api_version(DescribeConfigsRequest, max_version=2) if version == 0: if include_synonyms: raise IncompatibleBrokerVersion( @@ -881,6 +1009,14 @@ def describe_configs(self, config_resources, include_synonyms=False): @staticmethod def _convert_alter_config_resource_request(config_resource): + """Convert a ConfigResource into the format required by AlterConfigsRequest. + + Arguments: + config_resource: A ConfigResource with resource_type, name, and config (key, value) pairs. + + Returns: + A tuple: (resource_type, resource_name, [(config_key, config_value), ...]). + """ return ( config_resource.resource_type, config_resource.name, @@ -898,10 +1034,13 @@ def alter_configs(self, config_resources): least-loaded node. See the comment in the source code for details. We would happily accept a PR fixing this. - :param config_resources: A list of ConfigResource objects. - :return: Appropriate version of AlterConfigsResponse class. + Arguments: + config_resources: A list of ConfigResource objects. + + Returns: + Appropriate version of AlterConfigsResponse class. """ - version = self._matching_api_version(AlterConfigsRequest) + version = self._client.api_version(AlterConfigsRequest, max_version=1) if version <= 1: request = AlterConfigsRequest[version]( resources=[self._convert_alter_config_resource_request(config_resource) for config_resource in config_resources] @@ -930,6 +1069,15 @@ def alter_configs(self, config_resources): @staticmethod def _convert_create_partitions_request(topic_name, new_partitions): + """Convert a NewPartitions object into the tuple format for CreatePartitionsRequest. + + Arguments: + topic_name: The name of the existing topic. + new_partitions: A NewPartitions instance with total_count and new_assignments. + + Returns: + A tuple: (topic_name, (total_count, [list_of_assignments])). + """ return ( topic_name, ( @@ -941,14 +1089,19 @@ def _convert_create_partitions_request(topic_name, new_partitions): def create_partitions(self, topic_partitions, timeout_ms=None, validate_only=False): """Create additional partitions for an existing topic. - :param topic_partitions: A map of topic name strings to NewPartition objects. - :param timeout_ms: Milliseconds to wait for new partitions to be - created before the broker returns. - :param validate_only: If True, don't actually create new partitions. - Default: False - :return: Appropriate version of CreatePartitionsResponse class. + Arguments: + topic_partitions: A map of topic name strings to NewPartition objects. + + Keyword Arguments: + timeout_ms (numeric, optional): Milliseconds to wait for new partitions to be + created before the broker returns. + validate_only (bool, optional): If True, don't actually create new partitions. + Default: False + + Returns: + Appropriate version of CreatePartitionsResponse class. """ - version = self._matching_api_version(CreatePartitionsRequest) + version = self._client.api_version(CreatePartitionsRequest, max_version=1) timeout_ms = self._validate_timeout(timeout_ms) if version <= 1: request = CreatePartitionsRequest[version]( @@ -980,12 +1133,14 @@ def create_partitions(self, topic_partitions, timeout_ms=None, validate_only=Fal def _describe_consumer_groups_send_request(self, group_id, group_coordinator_id, include_authorized_operations=False): """Send a DescribeGroupsRequest to the group's coordinator. - :param group_id: The group name as a string - :param group_coordinator_id: The node_id of the groups' coordinator - broker. - :return: A message future. + Arguments: + group_id: The group name as a string + group_coordinator_id: The node_id of the groups' coordinator broker. + + Returns: + A message future. """ - version = self._matching_api_version(DescribeGroupsRequest) + version = self._client.api_version(DescribeGroupsRequest, max_version=3) if version <= 2: if include_authorized_operations: raise IncompatibleBrokerVersion( @@ -1066,18 +1221,23 @@ def describe_consumer_groups(self, group_ids, group_coordinator_id=None, include Any errors are immediately raised. - :param group_ids: A list of consumer group IDs. These are typically the - group names as strings. - :param group_coordinator_id: The node_id of the groups' coordinator - broker. If set to None, it will query the cluster for each group to - find that group's coordinator. Explicitly specifying this can be - useful for avoiding extra network round trips if you already know - the group coordinator. This is only useful when all the group_ids - have the same coordinator, otherwise it will error. Default: None. - :param include_authorized_operations: Whether or not to include - information about the operations a group is allowed to perform. - Only supported on API version >= v3. Default: False. - :return: A list of group descriptions. For now the group descriptions + Arguments: + group_ids: A list of consumer group IDs. These are typically the + group names as strings. + + Keyword Arguments: + group_coordinator_id (int, optional): The node_id of the groups' coordinator + broker. If set to None, it will query the cluster for each group to + find that group's coordinator. Explicitly specifying this can be + useful for avoiding extra network round trips if you already know + the group coordinator. This is only useful when all the group_ids + have the same coordinator, otherwise it will error. Default: None. + include_authorized_operations (bool, optional): Whether or not to include + information about the operations a group is allowed to perform. + Only supported on API version >= v3. Default: False. + + Returns: + A list of group descriptions. For now the group descriptions are the raw results from the DescribeGroupsResponse. Long-term, we plan to change this to return namedtuples as well as decoding the partition assignments. @@ -1108,10 +1268,13 @@ def describe_consumer_groups(self, group_ids, group_coordinator_id=None, include def _list_consumer_groups_send_request(self, broker_id): """Send a ListGroupsRequest to a broker. - :param broker_id: The broker's node_id. - :return: A message future + Arguments: + broker_id (int): The broker's node_id. + + Returns: + A message future """ - version = self._matching_api_version(ListGroupsRequest) + version = self._client.api_version(ListGroupsRequest, max_version=2) if version <= 2: request = ListGroupsRequest[version]() else: @@ -1149,15 +1312,20 @@ def list_consumer_groups(self, broker_ids=None): As soon as any error is encountered, it is immediately raised. - :param broker_ids: A list of broker node_ids to query for consumer - groups. If set to None, will query all brokers in the cluster. - Explicitly specifying broker(s) can be useful for determining which - consumer groups are coordinated by those broker(s). Default: None - :return list: List of tuples of Consumer Groups. - :exception GroupCoordinatorNotAvailableError: The coordinator is not - available, so cannot process requests. - :exception GroupLoadInProgressError: The coordinator is loading and - hence can't process requests. + Keyword Arguments: + broker_ids ([int], optional): A list of broker node_ids to query for consumer + groups. If set to None, will query all brokers in the cluster. + Explicitly specifying broker(s) can be useful for determining which + consumer groups are coordinated by those broker(s). Default: None + + Returns: + list: List of tuples of Consumer Groups. + + Raises: + GroupCoordinatorNotAvailableError: The coordinator is not + available, so cannot process requests. + GroupLoadInProgressError: The coordinator is loading and + hence can't process requests. """ # While we return a list, internally use a set to prevent duplicates # because if a group coordinator fails after being queried, and its @@ -1177,12 +1345,19 @@ def _list_consumer_group_offsets_send_request(self, group_id, group_coordinator_id, partitions=None): """Send an OffsetFetchRequest to a broker. - :param group_id: The consumer group id name for which to fetch offsets. - :param group_coordinator_id: The node_id of the group's coordinator - broker. - :return: A message future + Arguments: + group_id (str): The consumer group id name for which to fetch offsets. + group_coordinator_id (int): The node_id of the group's coordinator broker. + + Keyword Arguments: + partitions: A list of TopicPartitions for which to fetch + offsets. On brokers >= 0.10.2, this can be set to None to fetch all + known offsets for the consumer group. Default: None. + + Returns: + A message future """ - version = self._matching_api_version(OffsetFetchRequest) + version = self._client.api_version(OffsetFetchRequest, max_version=5) if version <= 3: if partitions is None: if version <= 1: @@ -1208,11 +1383,14 @@ def _list_consumer_group_offsets_send_request(self, group_id, def _list_consumer_group_offsets_process_response(self, response): """Process an OffsetFetchResponse. - :param response: an OffsetFetchResponse. - :return: A dictionary composed of TopicPartition keys and + Arguments: + response: an OffsetFetchResponse. + + Returns: + A dictionary composed of TopicPartition keys and OffsetAndMetadata values. """ - if response.API_VERSION <= 3: + if response.API_VERSION <= 5: # OffsetFetchResponse_v1 lacks a top-level error_code if response.API_VERSION > 1: @@ -1227,13 +1405,18 @@ def _list_consumer_group_offsets_process_response(self, response): # OffsetAndMetadata values--this is what the Java AdminClient returns offsets = {} for topic, partitions in response.topics: - for partition, offset, metadata, error_code in partitions: + for partition_data in partitions: + if response.API_VERSION <= 4: + partition, offset, metadata, error_code = partition_data + leader_epoch = -1 + else: + partition, offset, leader_epoch, metadata, error_code = partition_data error_type = Errors.for_code(error_code) if error_type is not Errors.NoError: raise error_type( "Unable to fetch consumer group offsets for topic {}, partition {}" .format(topic, partition)) - offsets[TopicPartition(topic, partition)] = OffsetAndMetadata(offset, metadata) + offsets[TopicPartition(topic, partition)] = OffsetAndMetadata(offset, metadata, leader_epoch) else: raise NotImplementedError( "Support for OffsetFetchResponse_v{} has not yet been added to KafkaAdminClient." @@ -1250,17 +1433,22 @@ def list_consumer_group_offsets(self, group_id, group_coordinator_id=None, As soon as any error is encountered, it is immediately raised. - :param group_id: The consumer group id name for which to fetch offsets. - :param group_coordinator_id: The node_id of the group's coordinator - broker. If set to None, will query the cluster to find the group - coordinator. Explicitly specifying this can be useful to prevent - that extra network round trip if you already know the group - coordinator. Default: None. - :param partitions: A list of TopicPartitions for which to fetch - offsets. On brokers >= 0.10.2, this can be set to None to fetch all - known offsets for the consumer group. Default: None. - :return dictionary: A dictionary with TopicPartition keys and - OffsetAndMetada values. Partitions that are not specified and for + Arguments: + group_id (str): The consumer group id name for which to fetch offsets. + + Keyword Arguments: + group_coordinator_id (int, optional): The node_id of the group's coordinator + broker. If set to None, will query the cluster to find the group + coordinator. Explicitly specifying this can be useful to prevent + that extra network round trip if you already know the group + coordinator. Default: None. + partitions: A list of TopicPartitions for which to fetch + offsets. On brokers >= 0.10.2, this can be set to None to fetch all + known offsets for the consumer group. Default: None. + + Returns: + dictionary: A dictionary with TopicPartition keys and + OffsetAndMetadata values. Partitions that are not specified and for which the group_id does not have a recorded offset are omitted. An offset value of `-1` indicates the group_id has no offset for that TopicPartition. A `-1` can only happen for partitions that are @@ -1283,14 +1471,19 @@ def delete_consumer_groups(self, group_ids, group_coordinator_id=None): The result needs checking for potential errors. - :param group_ids: The consumer group ids of the groups which are to be deleted. - :param group_coordinator_id: The node_id of the broker which is the coordinator for - all the groups. Use only if all groups are coordinated by the same broker. - If set to None, will query the cluster to find the coordinator for every single group. - Explicitly specifying this can be useful to prevent - that extra network round trips if you already know the group - coordinator. Default: None. - :return: A list of tuples (group_id, KafkaError) + Arguments: + group_ids ([str]): The consumer group ids of the groups which are to be deleted. + + Keyword Arguments: + group_coordinator_id (int, optional): The node_id of the broker which is + the coordinator for all the groups. Use only if all groups are coordinated + by the same broker. If set to None, will query the cluster to find the coordinator + for every single group. Explicitly specifying this can be useful to prevent + that extra network round trips if you already know the group coordinator. + Default: None. + + Returns: + A list of tuples (group_id, KafkaError) """ if group_coordinator_id is not None: futures = [self._delete_consumer_groups_send_request(group_ids, group_coordinator_id)] @@ -1311,6 +1504,14 @@ def delete_consumer_groups(self, group_ids, group_coordinator_id=None): return results def _convert_delete_groups_response(self, response): + """Parse the DeleteGroupsResponse, mapping group IDs to their respective errors. + + Arguments: + response: A DeleteGroupsResponse object from the broker. + + Returns: + A list of (group_id, KafkaError) for each deleted group. + """ if response.API_VERSION <= 1: results = [] for group_id, error_code in response.results: @@ -1322,14 +1523,16 @@ def _convert_delete_groups_response(self, response): .format(response.API_VERSION)) def _delete_consumer_groups_send_request(self, group_ids, group_coordinator_id): - """Send a DeleteGroups request to a broker. + """Send a DeleteGroupsRequest to the specified broker (the group coordinator). + + Arguments: + group_ids ([str]): A list of consumer group IDs to be deleted. + group_coordinator_id (int): The node_id of the broker coordinating these groups. - :param group_ids: The consumer group ids of the groups which are to be deleted. - :param group_coordinator_id: The node_id of the broker which is the coordinator for - all the groups. - :return: A message future + Returns: + A future representing the in-flight DeleteGroupsRequest. """ - version = self._matching_api_version(DeleteGroupsRequest) + version = self._client.api_version(DeleteGroupsRequest, max_version=1) if version <= 1: request = DeleteGroupsRequest[version](group_ids) else: @@ -1339,9 +1542,34 @@ def _delete_consumer_groups_send_request(self, group_ids, group_coordinator_id): return self._send_request_to_node(group_coordinator_id, request) def _wait_for_futures(self, futures): + """Block until all futures complete. If any fail, raise the encountered exception. + + Arguments: + futures: A list of Future objects awaiting results. + + Raises: + The first encountered exception if a future fails. + """ while not all(future.succeeded() for future in futures): for future in futures: self._client.poll(future=future) if future.failed(): raise future.exception # pylint: disable-msg=raising-bad-type + + def describe_log_dirs(self): + """Send a DescribeLogDirsRequest request to a broker. + + Returns: + A message future + """ + version = self._client.api_version(DescribeLogDirsRequest, max_version=0) + if version <= 0: + request = DescribeLogDirsRequest[version]() + future = self._send_request_to_node(self._client.least_loaded_node(), request) + self._wait_for_futures([future]) + else: + raise NotImplementedError( + "Support for DescribeLogDirsRequest_v{} has not yet been added to KafkaAdminClient." + .format(version)) + return future.value diff --git a/kafka/client_async.py b/kafka/client_async.py index 58f22d4ec..b72c05dac 100644 --- a/kafka/client_async.py +++ b/kafka/client_async.py @@ -19,12 +19,13 @@ from kafka.vendor import six from kafka.cluster import ClusterMetadata -from kafka.conn import BrokerConnection, ConnectionStates, collect_hosts, get_ip_port_afi +from kafka.conn import BrokerConnection, ConnectionStates, get_ip_port_afi from kafka import errors as Errors from kafka.future import Future from kafka.metrics import AnonMeasurable from kafka.metrics.stats import Avg, Count, Rate from kafka.metrics.stats.rate import TimeUnit +from kafka.protocol.broker_api_versions import BROKER_API_VERSIONS from kafka.protocol.metadata import MetadataRequest from kafka.util import Dict, WeakMethod # Although this looks unused, it actually monkey-patches socket.socketpair() @@ -75,7 +76,7 @@ class KafkaClient(object): reconnection attempts will continue periodically with this fixed rate. To avoid connection storms, a randomization factor of 0.2 will be applied to the backoff resulting in a random range between - 20% below and 20% above the computed value. Default: 1000. + 20% below and 20% above the computed value. Default: 30000. request_timeout_ms (int): Client request timeout in milliseconds. Default: 30000. connections_max_idle_ms: Close idle connections after the number of @@ -101,6 +102,9 @@ class KafkaClient(object): which we force a refresh of metadata even if we haven't seen any partition leadership changes to proactively discover any new brokers or partitions. Default: 300000 + allow_auto_create_topics (bool): Enable/disable auto topic creation + on metadata request. Only available with api_version >= (0, 11). + Default: True security_protocol (str): Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL. Default: PLAINTEXT. @@ -129,12 +133,24 @@ class KafkaClient(object): format. If no cipher can be selected (because compile-time options or other configuration forbids use of all the specified ciphers), an ssl.SSLError will be raised. See ssl.SSLContext.set_ciphers - api_version (tuple): Specify which Kafka API version to use. If set - to None, KafkaClient will attempt to infer the broker version by - probing various APIs. Example: (0, 10, 2). Default: None + api_version (tuple): Specify which Kafka API version to use. If set to + None, the client will attempt to determine the broker version via + ApiVersionsRequest API or, for brokers earlier than 0.10, probing + various known APIs. Dynamic version checking is performed eagerly + during __init__ and can raise NoBrokersAvailableError if no connection + was made before timeout (see api_version_auto_timeout_ms below). + Different versions enable different functionality. + + Examples: + (3, 9) most recent broker release, enable all supported features + (0, 10, 0) enables sasl authentication + (0, 8, 0) enables basic functionality only + + Default: None api_version_auto_timeout_ms (int): number of milliseconds to throw a timeout exception from the constructor when checking the broker - api version. Only applies if api_version is None + api version. Only applies if api_version set to None. + Default: 2000 selector (selectors.BaseSelector): Provide a specific selector implementation to use for I/O multiplexing. Default: selectors.DefaultSelector @@ -148,6 +164,9 @@ class KafkaClient(object): Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms. sasl_plain_password (str): password for sasl PLAIN and SCRAM authentication. Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms. + sasl_kerberos_name (str or gssapi.Name): Constructed gssapi.Name for use with + sasl mechanism handshake. If provided, sasl_kerberos_service_name and + sasl_kerberos_domain name are ignored. Default: None. sasl_kerberos_service_name (str): Service name to include in GSSAPI sasl mechanism handshake. Default: 'kafka' sasl_kerberos_domain_name (str): kerberos domain name to use in GSSAPI @@ -164,7 +183,7 @@ class KafkaClient(object): 'wakeup_timeout_ms': 3000, 'connections_max_idle_ms': 9 * 60 * 1000, 'reconnect_backoff_ms': 50, - 'reconnect_backoff_max_ms': 1000, + 'reconnect_backoff_max_ms': 30000, 'max_in_flight_requests_per_connection': 5, 'receive_buffer_bytes': None, 'send_buffer_bytes': None, @@ -172,6 +191,7 @@ class KafkaClient(object): 'sock_chunk_bytes': 4096, # undocumented experimental option 'sock_chunk_buffer_count': 1000, # undocumented experimental option 'retry_backoff_ms': 100, + 'allow_auto_create_topics': True, 'metadata_max_age_ms': 300000, 'security_protocol': 'PLAINTEXT', 'ssl_context': None, @@ -190,6 +210,7 @@ class KafkaClient(object): 'sasl_mechanism': None, 'sasl_plain_username': None, 'sasl_plain_password': None, + 'sasl_kerberos_name': None, 'sasl_kerberos_service_name': 'kafka', 'sasl_kerberos_domain_name': None, 'sasl_oauth_token_provider': None @@ -204,8 +225,9 @@ def __init__(self, **configs): # these properties need to be set on top of the initialization pipeline # because they are used when __del__ method is called self._closed = False - self._wake_r, self._wake_w = socket.socketpair() self._selector = self.config['selector']() + self._init_wakeup_socketpair() + self._wake_lock = threading.Lock() self.cluster = ClusterMetadata(**self.config) self._topics = set() # empty set will fetch all topic metadata @@ -215,11 +237,10 @@ def __init__(self, **configs): self._connecting = set() self._sending = set() self._refresh_on_disconnects = True + + # Not currently used, but data is collected internally self._last_bootstrap = 0 self._bootstrap_fails = 0 - self._wake_r.setblocking(False) - self._wake_w.settimeout(self.config['wakeup_timeout_ms'] / 1000.0) - self._wake_lock = threading.Lock() self._lock = threading.RLock() @@ -228,7 +249,6 @@ def __init__(self, **configs): # lock above. self._pending_completion = collections.deque() - self._selector.register(self._wake_r, selectors.EVENT_READ) self._idle_expiry_manager = IdleConnectionManager(self.config['connections_max_idle_ms']) self._sensors = None if self.config['metrics']: @@ -236,26 +256,47 @@ def __init__(self, **configs): self.config['metric_group_prefix'], weakref.proxy(self._conns)) - self._num_bootstrap_hosts = len(collect_hosts(self.config['bootstrap_servers'])) - # Check Broker Version if not set explicitly if self.config['api_version'] is None: - check_timeout = self.config['api_version_auto_timeout_ms'] / 1000 - self.config['api_version'] = self.check_version(timeout=check_timeout) - - def _can_bootstrap(self): - effective_failures = self._bootstrap_fails // self._num_bootstrap_hosts - backoff_factor = 2 ** effective_failures - backoff_ms = min(self.config['reconnect_backoff_ms'] * backoff_factor, - self.config['reconnect_backoff_max_ms']) + self.config['api_version'] = self.check_version() + elif self.config['api_version'] in BROKER_API_VERSIONS: + self._api_versions = BROKER_API_VERSIONS[self.config['api_version']] + elif (self.config['api_version'] + (0,)) in BROKER_API_VERSIONS: + log.warning('Configured api_version %s is ambiguous; using %s', + self.config['api_version'], self.config['api_version'] + (0,)) + self.config['api_version'] = self.config['api_version'] + (0,) + self._api_versions = BROKER_API_VERSIONS[self.config['api_version']] + else: + compatible_version = None + for v in sorted(BROKER_API_VERSIONS.keys(), reverse=True): + if v <= self.config['api_version']: + compatible_version = v + break + if compatible_version: + log.warning('Configured api_version %s not supported; using %s', + self.config['api_version'], compatible_version) + self._api_versions = BROKER_API_VERSIONS[compatible_version] + else: + raise Errors.UnrecognizedBrokerVersion(self.config['api_version']) - backoff_ms *= random.uniform(0.8, 1.2) + def _init_wakeup_socketpair(self): + self._wake_r, self._wake_w = socket.socketpair() + self._wake_r.setblocking(False) + self._wake_w.settimeout(self.config['wakeup_timeout_ms'] / 1000.0) + self._waking = False + self._selector.register(self._wake_r, selectors.EVENT_READ) - next_at = self._last_bootstrap + backoff_ms / 1000.0 - now = time.time() - if next_at > now: - return False - return True + def _close_wakeup_socketpair(self): + if self._wake_r is not None: + try: + self._selector.unregister(self._wake_r) + except (KeyError, ValueError, TypeError): + pass + self._wake_r.close() + if self._wake_w is not None: + self._wake_w.close() + self._wake_r = None + self._wake_w = None def _can_connect(self, node_id): if node_id not in self._conns: @@ -267,7 +308,7 @@ def _can_connect(self, node_id): def _conn_state_change(self, node_id, sock, conn): with self._lock: - if conn.connecting(): + if conn.state is ConnectionStates.CONNECTING: # SSL connections can enter this state 2x (second during Handshake) if node_id not in self._connecting: self._connecting.add(node_id) @@ -279,7 +320,19 @@ def _conn_state_change(self, node_id, sock, conn): if self.cluster.is_bootstrap(node_id): self._last_bootstrap = time.time() - elif conn.connected(): + elif conn.state is ConnectionStates.API_VERSIONS_SEND: + try: + self._selector.register(sock, selectors.EVENT_WRITE, conn) + except KeyError: + self._selector.modify(sock, selectors.EVENT_WRITE, conn) + + elif conn.state in (ConnectionStates.API_VERSIONS_RECV, ConnectionStates.AUTHENTICATING): + try: + self._selector.register(sock, selectors.EVENT_READ, conn) + except KeyError: + self._selector.modify(sock, selectors.EVENT_READ, conn) + + elif conn.state is ConnectionStates.CONNECTED: log.debug("Node %s connected", node_id) if node_id in self._connecting: self._connecting.remove(node_id) @@ -296,6 +349,8 @@ def _conn_state_change(self, node_id, sock, conn): if self.cluster.is_bootstrap(node_id): self._bootstrap_fails = 0 + if self._api_versions is None: + self._api_versions = conn._api_versions else: for node_id in list(self._conns.keys()): @@ -362,14 +417,24 @@ def _should_recycle_connection(self, conn): return False - def _maybe_connect(self, node_id): - """Idempotent non-blocking connection attempt to the given node id.""" + def _init_connect(self, node_id): + """Idempotent non-blocking connection attempt to the given node id. + + Returns True if connection object exists and is connected / connecting + """ with self._lock: conn = self._conns.get(node_id) + # Check if existing connection should be recreated because host/port changed + if conn is not None and self._should_recycle_connection(conn): + self._conns.pop(node_id).close() + conn = None + if conn is None: broker = self.cluster.broker_metadata(node_id) - assert broker, 'Broker id %s not in current metadata' % (node_id,) + if broker is None: + log.debug('Broker id %s not in current metadata', node_id) + return False log.debug("Initiating connection to node %s at %s:%s", node_id, broker.host, broker.port) @@ -381,16 +446,9 @@ def _maybe_connect(self, node_id): **self.config) self._conns[node_id] = conn - # Check if existing connection should be recreated because host/port changed - elif self._should_recycle_connection(conn): - self._conns.pop(node_id) - return False - - elif conn.connected(): - return True - - conn.connect() - return conn.connected() + if conn.disconnected(): + conn.connect() + return not conn.disconnected() def ready(self, node_id, metadata_priority=True): """Check whether a node is connected and ok to send more requests. @@ -416,8 +474,7 @@ def connected(self, node_id): def _close(self): if not self._closed: self._closed = True - self._wake_r.close() - self._wake_w.close() + self._close_wakeup_socketpair() self._selector.close() def close(self, node_id=None): @@ -464,9 +521,8 @@ def is_disconnected(self, node_id): def connection_delay(self, node_id): """ Return the number of milliseconds to wait, based on the connection - state, before attempting to send data. When disconnected, this respects - the reconnect backoff time. When connecting, returns 0 to allow - non-blocking connect to finish. When connected, returns a very large + state, before attempting to send data. When connecting or disconnected, + this respects the reconnect backoff time. When connected, returns a very large number to handle slow/stalled connections. Arguments: @@ -480,6 +536,16 @@ def connection_delay(self, node_id): return 0 return conn.connection_delay() + def throttle_delay(self, node_id): + """ + Return the number of milliseconds to wait until a broker is no longer throttled. + When disconnected / connecting, returns 0. + """ + conn = self._conns.get(node_id) + if conn is None: + return 0 + return conn.throttle_delay() + def is_ready(self, node_id, metadata_priority=True): """Check whether a node is ready to send more requests. @@ -512,7 +578,7 @@ def _can_send_request(self, node_id): return False return conn.connected() and conn.can_send_more() - def send(self, node_id, request, wakeup=True): + def send(self, node_id, request, wakeup=True, request_timeout_ms=None): """Send a request to a specific node. Bytes are placed on an internal per-connection send-queue. Actual network I/O will be triggered in a subsequent call to .poll() @@ -520,7 +586,13 @@ def send(self, node_id, request, wakeup=True): Arguments: node_id (int): destination node request (Struct): request object (not-encoded) - wakeup (bool): optional flag to disable thread-wakeup + + Keyword Arguments: + wakeup (bool, optional): optional flag to disable thread-wakeup. + request_timeout_ms (int, optional): Provide custom timeout in milliseconds. + If response is not processed before timeout, client will fail the + request and close the connection. + Default: None (uses value from client configuration) Raises: AssertionError: if node_id is not in current cluster metadata @@ -536,8 +608,9 @@ def send(self, node_id, request, wakeup=True): # conn.send will queue the request internally # we will need to call send_pending_requests() # to trigger network I/O - future = conn.send(request, blocking=False) - self._sending.add(conn) + future = conn.send(request, blocking=False, request_timeout_ms=request_timeout_ms) + if not future.is_done: + self._sending.add(conn) # Wakeup signal is useful in case another thread is # blocked waiting for incoming network traffic while holding @@ -563,9 +636,7 @@ def poll(self, timeout_ms=None, future=None): Returns: list: responses received (can be empty) """ - if future is not None: - timeout_ms = 100 - elif timeout_ms is None: + if timeout_ms is None: timeout_ms = self.config['request_timeout_ms'] elif not isinstance(timeout_ms, (int, float)): raise TypeError('Invalid type for timeout: %s' % type(timeout_ms)) @@ -579,9 +650,15 @@ def poll(self, timeout_ms=None, future=None): # Attempt to complete pending connections for node_id in list(self._connecting): - self._maybe_connect(node_id) - - # Send a metadata request if needed + # False return means no more connection progress is possible + # Connected nodes will update _connecting via state_change callback + if not self._init_connect(node_id): + # It's possible that the connection attempt triggered a state change + # but if not, make sure to remove from _connecting list + if node_id in self._connecting: + self._connecting.remove(node_id) + + # Send a metadata request if needed (or initiate new connection) metadata_timeout_ms = self._maybe_refresh_metadata() # If we got a future that is already done, don't block in _poll @@ -589,14 +666,13 @@ def poll(self, timeout_ms=None, future=None): timeout = 0 else: idle_connection_timeout_ms = self._idle_expiry_manager.next_check_ms() + request_timeout_ms = self._next_ifr_request_timeout_ms() + log.debug("Timeouts: user %f, metadata %f, idle connection %f, request %f", timeout_ms, metadata_timeout_ms, idle_connection_timeout_ms, request_timeout_ms) timeout = min( timeout_ms, metadata_timeout_ms, idle_connection_timeout_ms, - self.config['request_timeout_ms']) - # if there are no requests in flight, do not block longer than the retry backoff - if self.in_flight_request_count() == 0: - timeout = min(timeout, self.config['retry_backoff_ms']) + request_timeout_ms) timeout = max(0, timeout) # avoid negative timeouts self._poll(timeout / 1000) @@ -615,6 +691,8 @@ def poll(self, timeout_ms=None, future=None): def _register_send_sockets(self): while self._sending: conn = self._sending.pop() + if conn._sock is None: + continue try: key = self._selector.get_key(conn._sock) events = key.events | selectors.EVENT_WRITE @@ -623,6 +701,11 @@ def _register_send_sockets(self): self._selector.register(conn._sock, selectors.EVENT_WRITE, conn) def _poll(self, timeout): + # Python throws OverflowError if timeout is > 2147483647 milliseconds + # (though the param to selector.select is in seconds) + # so convert any too-large timeout to blocking + if timeout > 2147483: + timeout = None # This needs to be locked, but since it is only called from within the # locked section of poll(), there is no additional lock acquisition here processed = set() @@ -695,11 +778,13 @@ def _poll(self, timeout): for conn in six.itervalues(self._conns): if conn.requests_timed_out(): + timed_out = conn.timed_out_ifrs() + timeout_ms = (timed_out[0][2] - timed_out[0][1]) * 1000 log.warning('%s timed out after %s ms. Closing connection.', - conn, conn.config['request_timeout_ms']) + conn, timeout_ms) conn.close(error=Errors.RequestTimedOutError( 'Request timed out after %s ms' % - conn.config['request_timeout_ms'])) + timeout_ms)) if self._sensors: self._sensors.io_time.record((time.time() - end_select) * 1000000000) @@ -737,16 +822,17 @@ def _fire_pending_completed_requests(self): break future.success(response) responses.append(response) + return responses def least_loaded_node(self): """Choose the node with fewest outstanding requests, with fallbacks. - This method will prefer a node with an existing connection and no - in-flight-requests. If no such node is found, a node will be chosen - randomly from disconnected nodes that are not "blacked out" (i.e., + This method will prefer a node with an existing connection (not throttled) + with no in-flight-requests. If no such node is found, a node will be chosen + randomly from all nodes that are not throttled or "blacked out" (i.e., are not subject to a reconnect backoff). If no node metadata has been - obtained, will return a bootstrap node (subject to exponential backoff). + obtained, will return a bootstrap node. Returns: node_id or None if no suitable node was found @@ -758,11 +844,11 @@ def least_loaded_node(self): found = None for node_id in nodes: conn = self._conns.get(node_id) - connected = conn is not None and conn.connected() - blacked_out = conn is not None and conn.blacked_out() + connected = conn is not None and conn.connected() and conn.can_send_more() + blacked_out = conn is not None and (conn.blacked_out() or conn.throttled()) curr_inflight = len(conn.in_flight_requests) if conn is not None else 0 if connected and curr_inflight == 0: - # if we find an established connection + # if we find an established connection (not throttled) # with no in-flight requests, we can stop right away return node_id elif not blacked_out and curr_inflight < inflight: @@ -772,6 +858,24 @@ def least_loaded_node(self): return found + def _refresh_delay_ms(self, node_id): + conn = self._conns.get(node_id) + if conn is not None and conn.connected(): + return self.throttle_delay(node_id) + else: + return self.connection_delay(node_id) + + def least_loaded_node_refresh_ms(self): + """Return connection or throttle delay in milliseconds for next available node. + + This method is used primarily for retry/backoff during metadata refresh + during / after a cluster outage, in which there are no available nodes. + + Returns: + float: delay_ms + """ + return min([self._refresh_delay_ms(broker.nodeId) for broker in self.cluster.brokers()]) + def set_topics(self, topics): """Set specific topics to track for metadata. @@ -803,12 +907,18 @@ def add_topic(self, topic): self._topics.add(topic) return self.cluster.request_update() + def _next_ifr_request_timeout_ms(self): + if self._conns: + return min([conn.next_ifr_request_timeout_ms() for conn in six.itervalues(self._conns)]) + else: + return float('inf') + # This method should be locked when running multi-threaded def _maybe_refresh_metadata(self, wakeup=False): """Send a metadata request if needed. Returns: - int: milliseconds until next refresh + float: milliseconds until next refresh """ ttl = self.cluster.ttl() wait_for_in_progress_ms = self.config['request_timeout_ms'] if self._metadata_refresh_in_progress else 0 @@ -822,18 +932,42 @@ def _maybe_refresh_metadata(self, wakeup=False): # least_loaded_node() node_id = self.least_loaded_node() if node_id is None: - log.debug("Give up sending metadata request since no node is available"); - return self.config['reconnect_backoff_ms'] + next_connect_ms = self.least_loaded_node_refresh_ms() + log.debug("Give up sending metadata request since no node is available. (reconnect delay %d ms)", next_connect_ms) + return next_connect_ms + if not self._can_send_request(node_id): + # If there's any connection establishment underway, wait until it completes. This prevents + # the client from unnecessarily connecting to additional nodes while a previous connection + # attempt has not been completed. + if self._connecting: + return float('inf') + + elif self._can_connect(node_id): + log.debug("Initializing connection to node %s for metadata request", node_id) + self._connecting.add(node_id) + if not self._init_connect(node_id): + if node_id in self._connecting: + self._connecting.remove(node_id) + # Connection attempt failed immediately, need to retry with a different node + return self.config['reconnect_backoff_ms'] + else: + # Existing connection throttled or max in flight requests. + return self.throttle_delay(node_id) or self.config['request_timeout_ms'] + + # Recheck node_id in case we were able to connect immediately above if self._can_send_request(node_id): topics = list(self._topics) if not topics and self.cluster.is_bootstrap(node_id): topics = list(self.config['bootstrap_topics_filter']) + api_version = self.api_version(MetadataRequest, max_version=7) if self.cluster.need_all_topic_metadata or not topics: - topics = [] if self.config['api_version'] < (0, 10) else None - api_version = 0 if self.config['api_version'] < (0, 10) else 1 - request = MetadataRequest[api_version](topics) + topics = MetadataRequest[api_version].ALL_TOPICS + if api_version >= 4: + request = MetadataRequest[api_version](topics, self.config['allow_auto_create_topics']) + else: + request = MetadataRequest[api_version](topics) log.debug("Sending metadata request %s to node %s", request, node_id) future = self.send(node_id, request, wakeup=wakeup) future.add_callback(self.cluster.update_metadata) @@ -846,103 +980,144 @@ def refresh_done(val_or_error): future.add_errback(refresh_done) return self.config['request_timeout_ms'] - # If there's any connection establishment underway, wait until it completes. This prevents - # the client from unnecessarily connecting to additional nodes while a previous connection - # attempt has not been completed. + # Should only get here if still connecting if self._connecting: + return float('inf') + else: return self.config['reconnect_backoff_ms'] - if self.maybe_connect(node_id, wakeup=wakeup): - log.debug("Initializing connection to node %s for metadata request", node_id) - return self.config['reconnect_backoff_ms'] - - # connected but can't send more, OR connecting - # In either case we just need to wait for a network event - # to let us know the selected connection might be usable again. - return float('inf') - def get_api_versions(self): """Return the ApiVersions map, if available. - Note: A call to check_version must previously have succeeded and returned - version 0.10.0 or later + Note: Only available after bootstrap; requires broker version 0.10.0 or later. Returns: a map of dict mapping {api_key : (min_version, max_version)}, or None if ApiVersion is not supported by the kafka cluster. """ return self._api_versions - def check_version(self, node_id=None, timeout=2, strict=False): + def check_version(self, node_id=None, timeout=None, **kwargs): """Attempt to guess the version of a Kafka broker. - Note: It is possible that this method blocks longer than the - specified timeout. This can happen if the entire cluster - is down and the client enters a bootstrap backoff sleep. - This is only possible if node_id is None. + Keyword Arguments: + node_id (str, optional): Broker node id from cluster metadata. If None, attempts + to connect to any available broker until version is identified. + Default: None + timeout (num, optional): Maximum time in seconds to try to check broker version. + If unable to identify version before timeout, raise error (see below). + Default: api_version_auto_timeout_ms / 1000 - Returns: version tuple, i.e. (0, 10), (0, 9), (0, 8, 2), ... + Returns: version tuple, i.e. (3, 9), (2, 0), (0, 10, 2) etc Raises: NodeNotReadyError (if node_id is provided) NoBrokersAvailable (if node_id is None) - UnrecognizedBrokerVersion: please file bug if seen! - AssertionError (if strict=True): please file bug if seen! """ - self._lock.acquire() - end = time.time() + timeout - while time.time() < end: - - # It is possible that least_loaded_node falls back to bootstrap, - # which can block for an increasing backoff period - try_node = node_id or self.least_loaded_node() - if try_node is None: - self._lock.release() - raise Errors.NoBrokersAvailable() - self._maybe_connect(try_node) - conn = self._conns[try_node] - - # We will intentionally cause socket failures - # These should not trigger metadata refresh - self._refresh_on_disconnects = False - try: - remaining = end - time.time() - version = conn.check_version(timeout=remaining, strict=strict, topics=list(self.config['bootstrap_topics_filter'])) - if version >= (0, 10, 0): - # cache the api versions map if it's available (starting - # in 0.10 cluster version) - self._api_versions = conn.get_api_versions() - self._lock.release() - return version - except Errors.NodeNotReadyError: - # Only raise to user if this is a node-specific request + timeout = timeout or (self.config['api_version_auto_timeout_ms'] / 1000) + with self._lock: + end = time.time() + timeout + while time.time() < end: + time_remaining = max(end - time.time(), 0) + if node_id is not None and self.connection_delay(node_id) > 0: + sleep_time = min(time_remaining, self.connection_delay(node_id) / 1000.0) + if sleep_time > 0: + time.sleep(sleep_time) + continue + try_node = node_id or self.least_loaded_node() + if try_node is None: + sleep_time = min(time_remaining, self.least_loaded_node_refresh_ms() / 1000.0) + if sleep_time > 0: + log.warning('No node available during check_version; sleeping %.2f secs', sleep_time) + time.sleep(sleep_time) + continue + log.debug('Attempting to check version with node %s', try_node) + if not self._init_connect(try_node): + if try_node == node_id: + raise Errors.NodeNotReadyError("Connection failed to %s" % node_id) + else: + continue + conn = self._conns[try_node] + + while conn.connecting() and time.time() < end: + timeout_ms = min((end - time.time()) * 1000, 200) + self.poll(timeout_ms=timeout_ms) + + if conn._api_version is not None: + return conn._api_version + + # Timeout + else: if node_id is not None: - self._lock.release() - raise - finally: - self._refresh_on_disconnects = True + raise Errors.NodeNotReadyError(node_id) + else: + raise Errors.NoBrokersAvailable() - # Timeout - else: - self._lock.release() - raise Errors.NoBrokersAvailable() + def api_version(self, operation, max_version=None): + """Find the latest version of the protocol operation supported by both + this library and the broker. + + This resolves to the lesser of either the latest api version this + library supports, or the max version supported by the broker. + + Arguments: + operation: A list of protocol operation versions from kafka.protocol. + + Keyword Arguments: + max_version (int, optional): Provide an alternate maximum api version + to reflect limitations in user code. + + Returns: + int: The highest api version number compatible between client and broker. + + Raises: IncompatibleBrokerVersion if no matching version is found + """ + # Cap max_version at the largest available version in operation list + max_version = min(len(operation) - 1, max_version if max_version is not None else float('inf')) + broker_api_versions = self._api_versions + api_key = operation[0].API_KEY + if broker_api_versions is None or api_key not in broker_api_versions: + raise Errors.IncompatibleBrokerVersion( + "Kafka broker does not support the '{}' Kafka protocol." + .format(operation[0].__name__)) + broker_min_version, broker_max_version = broker_api_versions[api_key] + version = min(max_version, broker_max_version) + if version < broker_min_version: + # max library version is less than min broker version. Currently, + # no Kafka versions specify a min msg version. Maybe in the future? + raise Errors.IncompatibleBrokerVersion( + "No version of the '{}' Kafka protocol is supported by both the client and broker." + .format(operation[0].__name__)) + return version def wakeup(self): + if self._waking or self._wake_w is None: + return with self._wake_lock: try: self._wake_w.sendall(b'x') - except socket.timeout: + self._waking = True + except socket.timeout as e: log.warning('Timeout to send to wakeup socket!') - raise Errors.KafkaTimeoutError() - except socket.error: - log.warning('Unable to send to wakeup socket!') + raise Errors.KafkaTimeoutError(e) + except socket.error as e: + log.warning('Unable to send to wakeup socket! %s', e) + raise e def _clear_wake_fd(self): # reading from wake socket should only happen in a single thread - while True: - try: - self._wake_r.recv(1024) - except socket.error: - break + with self._wake_lock: + self._waking = False + while True: + try: + if not self._wake_r.recv(1024): + # Non-blocking socket returns empty on error + log.warning("Error reading wakeup socket. Rebuilding socketpair.") + self._close_wakeup_socketpair() + self._init_wakeup_socketpair() + break + except socket.error: + # Non-blocking socket raises when socket is ok but no data available to read + break def _maybe_close_oldest_connection(self): expired_connection = self._idle_expiry_manager.poll_expired_connection() diff --git a/kafka/cluster.py b/kafka/cluster.py index 438baf29d..c28d36d20 100644 --- a/kafka/cluster.py +++ b/kafka/cluster.py @@ -21,7 +21,7 @@ class ClusterMetadata(object): A class to manage kafka cluster metadata. This class does not perform any IO. It simply updates internal state - given API responses (MetadataResponse, GroupCoordinatorResponse). + given API responses (MetadataResponse, FindCoordinatorResponse). Keyword Arguments: retry_backoff_ms (int): Milliseconds to backoff when retrying on @@ -58,6 +58,7 @@ def __init__(self, **configs): self.unauthorized_topics = set() self.internal_topics = set() self.controller = None + self.cluster_id = None self.config = copy.copy(self.DEFAULT_CONFIG) for key in self.config: @@ -140,6 +141,9 @@ def leader_for_partition(self, partition): return None return self._partitions[partition.topic][partition.partition].leader + def leader_epoch_for_partition(self, partition): + return self._partitions[partition.topic][partition.partition].leader_epoch + def partitions_for_broker(self, broker_id): """Return TopicPartitions for which the broker is a leader. @@ -236,7 +240,7 @@ def update_metadata(self, metadata): """ # In the common case where we ask for a single topic and get back an # error, we should fail the future - if len(metadata.topics) == 1 and metadata.topics[0][0] != 0: + if len(metadata.topics) == 1 and metadata.topics[0][0] != Errors.NoError.errno: error_code, topic = metadata.topics[0][:2] error = Errors.for_code(error_code)(topic) return self.failed_update(error) @@ -261,6 +265,11 @@ def update_metadata(self, metadata): else: _new_controller = _new_brokers.get(metadata.controller_id) + if metadata.API_VERSION < 2: + _new_cluster_id = None + else: + _new_cluster_id = metadata.cluster_id + _new_partitions = {} _new_broker_partitions = collections.defaultdict(set) _new_unauthorized_topics = set() @@ -277,10 +286,21 @@ def update_metadata(self, metadata): error_type = Errors.for_code(error_code) if error_type is Errors.NoError: _new_partitions[topic] = {} - for p_error, partition, leader, replicas, isr in partitions: + for partition_data in partitions: + leader_epoch = -1 + offline_replicas = [] + if metadata.API_VERSION >= 7: + p_error, partition, leader, leader_epoch, replicas, isr, offline_replicas = partition_data + elif metadata.API_VERSION >= 5: + p_error, partition, leader, replicas, isr, offline_replicas = partition_data + else: + p_error, partition, leader, replicas, isr = partition_data + _new_partitions[topic][partition] = PartitionMetadata( - topic=topic, partition=partition, leader=leader, - replicas=replicas, isr=isr, error=p_error) + topic=topic, partition=partition, + leader=leader, leader_epoch=leader_epoch, + replicas=replicas, isr=isr, offline_replicas=offline_replicas, + error=p_error) if leader != -1: _new_broker_partitions[leader].add( TopicPartition(topic, partition)) @@ -306,6 +326,7 @@ def update_metadata(self, metadata): with self._lock: self._brokers = _new_brokers self.controller = _new_controller + self.cluster_id = _new_cluster_id self._partitions = _new_partitions self._broker_partitions = _new_broker_partitions self.unauthorized_topics = _new_unauthorized_topics @@ -346,8 +367,8 @@ def add_group_coordinator(self, group, response): """Update with metadata for a group coordinator Arguments: - group (str): name of group from GroupCoordinatorRequest - response (GroupCoordinatorResponse): broker response + group (str): name of group from FindCoordinatorRequest + response (FindCoordinatorResponse): broker response Returns: string: coordinator node_id if metadata is updated, None on error @@ -355,7 +376,7 @@ def add_group_coordinator(self, group, response): log.debug("Updating coordinator for %s: %s", group, response) error_type = Errors.for_code(response.error_code) if error_type is not Errors.NoError: - log.error("GroupCoordinatorResponse error: %s", error_type) + log.error("FindCoordinatorResponse error: %s", error_type) self._groups[group] = -1 return diff --git a/kafka/codec.py b/kafka/codec.py index c740a181c..b73df060d 100644 --- a/kafka/codec.py +++ b/kafka/codec.py @@ -193,8 +193,15 @@ def _detect_xerial_stream(payload): """ if len(payload) > 16: - header = struct.unpack('!' + _XERIAL_V1_FORMAT, bytes(payload)[:16]) - return header == _XERIAL_V1_HEADER + magic = struct.unpack('!' + _XERIAL_V1_FORMAT[:8], bytes(payload)[:8]) + version, compat = struct.unpack('!' + _XERIAL_V1_FORMAT[8:], bytes(payload)[8:16]) + # Until there is more than one way to do xerial blocking, the version + compat + # fields can be ignored. Also some producers (i.e., redpanda) are known to + # incorrectly encode these as little-endian, and that causes us to fail decoding + # when we otherwise would have succeeded. + # See https://github.com/dpkp/kafka-python/issues/2414 + if magic == _XERIAL_V1_HEADER[:8]: + return True return False diff --git a/kafka/conn.py b/kafka/conn.py index 1efb8a0a1..6992bb5c2 100644 --- a/kafka/conn.py +++ b/kafka/conn.py @@ -14,7 +14,6 @@ from kafka.vendor import selectors34 as selectors import socket -import struct import threading import time @@ -23,16 +22,20 @@ import kafka.errors as Errors from kafka.future import Future from kafka.metrics.stats import Avg, Count, Max, Rate -from kafka.oauth.abstract import AbstractTokenProvider -from kafka.protocol.admin import SaslHandShakeRequest, DescribeAclsRequest_v2, DescribeClientQuotasRequest +from kafka.protocol.admin import DescribeAclsRequest, DescribeClientQuotasRequest, ListGroupsRequest +from kafka.protocol.api_versions import ApiVersionsRequest +from kafka.protocol.broker_api_versions import BROKER_API_VERSIONS from kafka.protocol.commit import OffsetFetchRequest -from kafka.protocol.offset import OffsetRequest -from kafka.protocol.produce import ProduceRequest -from kafka.protocol.metadata import MetadataRequest from kafka.protocol.fetch import FetchRequest +from kafka.protocol.find_coordinator import FindCoordinatorRequest +from kafka.protocol.list_offsets import ListOffsetsRequest +from kafka.protocol.metadata import MetadataRequest from kafka.protocol.parser import KafkaProtocol -from kafka.protocol.types import Int32, Int8 -from kafka.scram import ScramClient +from kafka.protocol.produce import ProduceRequest +from kafka.protocol.sasl_authenticate import SaslAuthenticateRequest +from kafka.protocol.sasl_handshake import SaslHandshakeRequest +from kafka.protocol.types import Int32 +from kafka.sasl import get_sasl_mechanism from kafka.version import __version__ @@ -45,10 +48,6 @@ DEFAULT_KAFKA_PORT = 9092 -SASL_QOP_AUTH = 1 -SASL_QOP_AUTH_INT = 2 -SASL_QOP_AUTH_CONF = 4 - try: import ssl ssl_available = True @@ -74,15 +73,6 @@ class SSLWantReadError(Exception): class SSLWantWriteError(Exception): pass -# needed for SASL_GSSAPI authentication: -try: - import gssapi - from gssapi.raw.misc import GSSError -except (ImportError, OSError): - #no gssapi available, will disable gssapi mechanism - gssapi = None - GSSError = None - AFI_NAMES = { socket.AF_UNSPEC: "unspecified", @@ -92,12 +82,13 @@ class SSLWantWriteError(Exception): class ConnectionStates(object): - DISCONNECTING = '' DISCONNECTED = '' CONNECTING = '' HANDSHAKE = '' CONNECTED = '' AUTHENTICATING = '' + API_VERSIONS_SEND = '' + API_VERSIONS_RECV = '' class BrokerConnection(object): @@ -120,7 +111,7 @@ class BrokerConnection(object): reconnection attempts will continue periodically with this fixed rate. To avoid connection storms, a randomization factor of 0.2 will be applied to the backoff resulting in a random range between - 20% below and 20% above the computed value. Default: 1000. + 20% below and 20% above the computed value. Default: 30000. request_timeout_ms (int): Client request timeout in milliseconds. Default: 30000. max_in_flight_requests_per_connection (int): Requests are pipelined @@ -165,11 +156,11 @@ class BrokerConnection(object): or other configuration forbids use of all the specified ciphers), an ssl.SSLError will be raised. See ssl.SSLContext.set_ciphers api_version (tuple): Specify which Kafka API version to use. - Accepted values are: (0, 8, 0), (0, 8, 1), (0, 8, 2), (0, 9), - (0, 10). Default: (0, 8, 2) + Must be None or >= (0, 10, 0) to enable SASL authentication. + Default: None api_version_auto_timeout_ms (int): number of milliseconds to throw a timeout exception from the constructor when checking the broker - api version. Only applies if api_version is None + api version. Only applies if api_version is None. Default: 2000. selector (selectors.BaseSelector): Provide a specific selector implementation to use for I/O multiplexing. Default: selectors.DefaultSelector @@ -185,6 +176,9 @@ class BrokerConnection(object): Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms. sasl_plain_password (str): password for sasl PLAIN and SCRAM authentication. Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms. + sasl_kerberos_name (str or gssapi.Name): Constructed gssapi.Name for use with + sasl mechanism handshake. If provided, sasl_kerberos_service_name and + sasl_kerberos_domain name are ignored. Default: None. sasl_kerberos_service_name (str): Service name to include in GSSAPI sasl mechanism handshake. Default: 'kafka' sasl_kerberos_domain_name (str): kerberos domain name to use in GSSAPI @@ -198,7 +192,7 @@ class BrokerConnection(object): 'node_id': 0, 'request_timeout_ms': 30000, 'reconnect_backoff_ms': 50, - 'reconnect_backoff_max_ms': 1000, + 'reconnect_backoff_max_ms': 30000, 'max_in_flight_requests_per_connection': 5, 'receive_buffer_bytes': None, 'send_buffer_bytes': None, @@ -214,7 +208,8 @@ class BrokerConnection(object): 'ssl_crlfile': None, 'ssl_password': None, 'ssl_ciphers': None, - 'api_version': (0, 8, 2), # default to most restrictive + 'api_version': None, + 'api_version_auto_timeout_ms': 2000, 'selector': selectors.DefaultSelector, 'state_change_callback': lambda node_id, sock, conn: True, 'metrics': None, @@ -222,12 +217,18 @@ class BrokerConnection(object): 'sasl_mechanism': None, 'sasl_plain_username': None, 'sasl_plain_password': None, + 'sasl_kerberos_name': None, 'sasl_kerberos_service_name': 'kafka', 'sasl_kerberos_domain_name': None, 'sasl_oauth_token_provider': None } SECURITY_PROTOCOLS = ('PLAINTEXT', 'SSL', 'SASL_PLAINTEXT', 'SASL_SSL') - SASL_MECHANISMS = ('PLAIN', 'GSSAPI', 'OAUTHBEARER', "SCRAM-SHA-256", "SCRAM-SHA-512") + VERSION_CHECKS = ( + ((0, 9), ListGroupsRequest[0]()), + ((0, 8, 2), FindCoordinatorRequest[0]('kafka-python-default-group')), + ((0, 8, 1), OffsetFetchRequest[0]('kafka-python-default-group', [])), + ((0, 8, 0), MetadataRequest[0]([])), + ) def __init__(self, host, port, afi, **configs): self.host = host @@ -236,6 +237,10 @@ def __init__(self, host, port, afi, **configs): self._sock_afi = afi self._sock_addr = None self._api_versions = None + self._api_version = None + self._check_version_idx = None + self._api_versions_idx = 2 + self._throttle_time = None self.config = copy.copy(self.DEFAULT_CONFIG) for key in self.config: @@ -256,26 +261,13 @@ def __init__(self, host, port, afi, **configs): assert self.config['security_protocol'] in self.SECURITY_PROTOCOLS, ( 'security_protocol must be in ' + ', '.join(self.SECURITY_PROTOCOLS)) + self._sasl_mechanism = None if self.config['security_protocol'] in ('SSL', 'SASL_SSL'): assert ssl_available, "Python wasn't built with SSL support" if self.config['security_protocol'] in ('SASL_PLAINTEXT', 'SASL_SSL'): - assert self.config['sasl_mechanism'] in self.SASL_MECHANISMS, ( - 'sasl_mechanism must be in ' + ', '.join(self.SASL_MECHANISMS)) - if self.config['sasl_mechanism'] in ('PLAIN', 'SCRAM-SHA-256', 'SCRAM-SHA-512'): - assert self.config['sasl_plain_username'] is not None, ( - 'sasl_plain_username required for PLAIN or SCRAM sasl' - ) - assert self.config['sasl_plain_password'] is not None, ( - 'sasl_plain_password required for PLAIN or SCRAM sasl' - ) - if self.config['sasl_mechanism'] == 'GSSAPI': - assert gssapi is not None, 'GSSAPI lib not available' - assert self.config['sasl_kerberos_service_name'] is not None, 'sasl_kerberos_service_name required for GSSAPI sasl' - if self.config['sasl_mechanism'] == 'OAUTHBEARER': - token_provider = self.config['sasl_oauth_token_provider'] - assert token_provider is not None, 'sasl_oauth_token_provider required for OAUTHBEARER sasl' - assert callable(getattr(token_provider, "token", None)), 'sasl_oauth_token_provider must implement method #token()' + self._sasl_mechanism = get_sasl_mechanism(self.config['sasl_mechanism'])(**self.config) + # This is not a general lock / this class is not generally thread-safe yet # However, to avoid pushing responsibility for maintaining # per-connection locks to the upstream client, we will use this lock to @@ -300,6 +292,7 @@ def __init__(self, host, port, afi, **configs): self._ssl_context = None if self.config['ssl_context'] is not None: self._ssl_context = self.config['ssl_context'] + self._api_versions_future = None self._sasl_auth_future = None self.last_attempt = 0 self._gai = [] @@ -368,7 +361,11 @@ def connect(self): log.debug('%s: creating new socket', self) assert self._sock is None self._sock_afi, self._sock_addr = next_lookup - self._sock = socket.socket(self._sock_afi, socket.SOCK_STREAM) + try: + self._sock = socket.socket(self._sock_afi, socket.SOCK_STREAM) + except (socket.error, OSError) as e: + self.close(e) + return self.state for option in self.config['socket_options']: log.debug('%s: setting socket option %s', self, option) @@ -399,17 +396,9 @@ def connect(self): self.config['state_change_callback'](self.node_id, self._sock, self) # _wrap_ssl can alter the connection state -- disconnects on failure self._wrap_ssl() - - elif self.config['security_protocol'] == 'SASL_PLAINTEXT': - log.debug('%s: initiating SASL authentication', self) - self.state = ConnectionStates.AUTHENTICATING - self.config['state_change_callback'](self.node_id, self._sock, self) - else: - # security_protocol PLAINTEXT - log.info('%s: Connection complete.', self) - self.state = ConnectionStates.CONNECTED - self._reset_reconnect_backoff() + log.debug('%s: checking broker Api Versions', self) + self.state = ConnectionStates.API_VERSIONS_SEND self.config['state_change_callback'](self.node_id, self._sock, self) # Connection failed @@ -428,15 +417,25 @@ def connect(self): if self.state is ConnectionStates.HANDSHAKE: if self._try_handshake(): log.debug('%s: completed SSL handshake.', self) - if self.config['security_protocol'] == 'SASL_SSL': - log.debug('%s: initiating SASL authentication', self) - self.state = ConnectionStates.AUTHENTICATING - else: - log.info('%s: Connection complete.', self) - self.state = ConnectionStates.CONNECTED - self._reset_reconnect_backoff() + log.debug('%s: checking broker Api Versions', self) + self.state = ConnectionStates.API_VERSIONS_SEND self.config['state_change_callback'](self.node_id, self._sock, self) + if self.state in (ConnectionStates.API_VERSIONS_SEND, ConnectionStates.API_VERSIONS_RECV): + if self._try_api_versions_check(): + # _try_api_versions_check has side-effects: possibly disconnected on socket errors + if self.state in (ConnectionStates.API_VERSIONS_SEND, ConnectionStates.API_VERSIONS_RECV): + if self.config['security_protocol'] in ('SASL_PLAINTEXT', 'SASL_SSL'): + log.debug('%s: initiating SASL authentication', self) + self.state = ConnectionStates.AUTHENTICATING + self.config['state_change_callback'](self.node_id, self._sock, self) + else: + # security_protocol PLAINTEXT + log.info('%s: Connection complete.', self) + self.state = ConnectionStates.CONNECTED + self._reset_reconnect_backoff() + self.config['state_change_callback'](self.node_id, self._sock, self) + if self.state is ConnectionStates.AUTHENTICATING: assert self.config['security_protocol'] in ('SASL_PLAINTEXT', 'SASL_SSL') if self._try_authenticate(): @@ -496,7 +495,7 @@ def _wrap_ssl(self): try: self._sock = self._ssl_context.wrap_socket( self._sock, - server_hostname=self.host, + server_hostname=self.host.rstrip("."), do_handshake_on_connect=False) except ssl.SSLError as e: log.exception('%s: Failed to wrap socket in SSLContext!', self) @@ -517,14 +516,105 @@ def _try_handshake(self): return False - def _try_authenticate(self): - assert self.config['api_version'] is None or self.config['api_version'] >= (0, 10) + def _try_api_versions_check(self): + if self._api_versions_future is None: + if self.config['api_version'] is not None: + self._api_version = self.config['api_version'] + self._api_versions = BROKER_API_VERSIONS[self._api_version] + return True + elif self._check_version_idx is None: + request = ApiVersionsRequest[self._api_versions_idx]() + future = Future() + response = self._send(request, blocking=True, request_timeout_ms=(self.config['api_version_auto_timeout_ms'] * 0.8)) + response.add_callback(self._handle_api_versions_response, future) + response.add_errback(self._handle_api_versions_failure, future) + self._api_versions_future = future + self.state = ConnectionStates.API_VERSIONS_RECV + self.config['state_change_callback'](self.node_id, self._sock, self) + elif self._check_version_idx < len(self.VERSION_CHECKS): + version, request = self.VERSION_CHECKS[self._check_version_idx] + future = Future() + response = self._send(request, blocking=True, request_timeout_ms=(self.config['api_version_auto_timeout_ms'] * 0.8)) + response.add_callback(self._handle_check_version_response, future, version) + response.add_errback(self._handle_check_version_failure, future) + self._api_versions_future = future + self.state = ConnectionStates.API_VERSIONS_RECV + self.config['state_change_callback'](self.node_id, self._sock, self) + else: + raise 'Unable to determine broker version.' + + for r, f in self.recv(): + f.success(r) + + # A connection error during blocking send could trigger close() which will reset the future + if self._api_versions_future is None: + return False + elif self._api_versions_future.failed(): + ex = self._api_versions_future.exception + if not isinstance(ex, Errors.KafkaConnectionError): + raise ex + return self._api_versions_future.succeeded() + def _handle_api_versions_response(self, future, response): + error_type = Errors.for_code(response.error_code) + # if error_type i UNSUPPORTED_VERSION: retry w/ latest version from response + if error_type is not Errors.NoError: + future.failure(error_type()) + if error_type is Errors.UnsupportedVersionError: + self._api_versions_idx -= 1 + if self._api_versions_idx >= 0: + self._api_versions_future = None + self.state = ConnectionStates.API_VERSIONS_SEND + self.config['state_change_callback'](self.node_id, self._sock, self) + else: + self.close(error=error_type()) + return + self._api_versions = dict([ + (api_key, (min_version, max_version)) + for api_key, min_version, max_version in response.api_versions + ]) + self._api_version = self._infer_broker_version_from_api_versions(self._api_versions) + log.info('Broker version identified as %s', '.'.join(map(str, self._api_version))) + future.success(self._api_version) + self.connect() + + def _handle_api_versions_failure(self, future, ex): + future.failure(ex) + self._check_version_idx = 0 + # after failure connection is closed, so state should already be DISCONNECTED + + def _handle_check_version_response(self, future, version, _response): + log.info('Broker version identified as %s', '.'.join(map(str, version))) + log.info('Set configuration api_version=%s to skip auto' + ' check_version requests on startup', version) + self._api_versions = BROKER_API_VERSIONS[version] + self._api_version = version + future.success(version) + self.connect() + + def _handle_check_version_failure(self, future, ex): + future.failure(ex) + self._check_version_idx += 1 + # after failure connection is closed, so state should already be DISCONNECTED + + def _sasl_handshake_version(self): + if self._api_versions is None: + raise RuntimeError('_api_versions not set') + if SaslHandshakeRequest[0].API_KEY not in self._api_versions: + raise Errors.UnsupportedVersionError('SaslHandshake') + + # Build a SaslHandshakeRequest message + min_version, max_version = self._api_versions[SaslHandshakeRequest[0].API_KEY] + if min_version > 1: + raise Errors.UnsupportedVersionError('SaslHandshake %s' % min_version) + return min(max_version, 1) + + def _try_authenticate(self): if self._sasl_auth_future is None: - # Build a SaslHandShakeRequest message - request = SaslHandShakeRequest[0](self.config['sasl_mechanism']) + version = self._sasl_handshake_version() + request = SaslHandshakeRequest[version](self.config['sasl_mechanism']) future = Future() - sasl_response = self._send(request) + sasl_response = self._send(request, blocking=True) sasl_response.add_callback(self._handle_sasl_handshake_response, future) sasl_response.add_errback(lambda f, e: f.failure(e), future) self._sasl_auth_future = future @@ -549,23 +639,18 @@ def _handle_sasl_handshake_response(self, future, response): return future.failure(error_type(self)) if self.config['sasl_mechanism'] not in response.enabled_mechanisms: - return future.failure( + future.failure( Errors.UnsupportedSaslMechanismError( 'Kafka broker does not support %s sasl mechanism. Enabled mechanisms are: %s' % (self.config['sasl_mechanism'], response.enabled_mechanisms))) - elif self.config['sasl_mechanism'] == 'PLAIN': - return self._try_authenticate_plain(future) - elif self.config['sasl_mechanism'] == 'GSSAPI': - return self._try_authenticate_gssapi(future) - elif self.config['sasl_mechanism'] == 'OAUTHBEARER': - return self._try_authenticate_oauth(future) - elif self.config['sasl_mechanism'].startswith("SCRAM-SHA-"): - return self._try_authenticate_scram(future) else: - return future.failure( - Errors.UnsupportedSaslMechanismError( - 'kafka-python does not support SASL mechanism %s' % - self.config['sasl_mechanism'])) + self._sasl_authenticate(future) + + assert future.is_done, 'SASL future not complete after mechanism processing!' + if future.failed(): + self.close(error=future.exception) + else: + self.connect() def _send_bytes(self, data): """Send some data via non-blocking IO @@ -619,224 +704,72 @@ def _recv_bytes_blocking(self, n): finally: self._sock.settimeout(0.0) - def _try_authenticate_plain(self, future): - if self.config['security_protocol'] == 'SASL_PLAINTEXT': - log.warning('%s: Sending username and password in the clear', self) - - data = b'' - # Send PLAIN credentials per RFC-4616 - msg = bytes('\0'.join([self.config['sasl_plain_username'], - self.config['sasl_plain_username'], - self.config['sasl_plain_password']]).encode('utf-8')) - size = Int32.encode(len(msg)) - - err = None - close = False - with self._lock: - if not self._can_send_recv(): - err = Errors.NodeNotReadyError(str(self)) - close = False - else: - try: - self._send_bytes_blocking(size + msg) - - # The server will send a zero sized message (that is Int32(0)) on success. - # The connection is closed on failure - data = self._recv_bytes_blocking(4) - - except (ConnectionError, TimeoutError) as e: - log.exception("%s: Error receiving reply from server", self) - err = Errors.KafkaConnectionError("%s: %s" % (self, e)) - close = True - - if err is not None: - if close: + def _send_sasl_authenticate(self, sasl_auth_bytes): + version = self._sasl_handshake_version() + if version == 1: + request = SaslAuthenticateRequest[0](sasl_auth_bytes) + self._send(request, blocking=True) + else: + try: + self._send_bytes_blocking(Int32.encode(len(sasl_auth_bytes)) + sasl_auth_bytes) + except (ConnectionError, TimeoutError) as e: + log.exception("%s: Error sending sasl auth bytes to server", self) + err = Errors.KafkaConnectionError("%s: %s" % (self, e)) self.close(error=err) - return future.failure(err) - - if data != b'\x00\x00\x00\x00': - error = Errors.AuthenticationFailedError('Unrecognized response during authentication') - return future.failure(error) - log.info('%s: Authenticated as %s via PLAIN', self, self.config['sasl_plain_username']) - return future.success(True) + def _recv_sasl_authenticate(self): + version = self._sasl_handshake_version() + # GSSAPI mechanism does not get a final recv in old non-framed mode + if version == 0 and self._sasl_mechanism.is_done(): + return b'' - def _try_authenticate_scram(self, future): - if self.config['security_protocol'] == 'SASL_PLAINTEXT': - log.warning('%s: Exchanging credentials in the clear', self) - - scram_client = ScramClient( - self.config['sasl_plain_username'], self.config['sasl_plain_password'], self.config['sasl_mechanism'] - ) - - err = None - close = False - with self._lock: - if not self._can_send_recv(): - err = Errors.NodeNotReadyError(str(self)) - close = False - else: - try: - client_first = scram_client.first_message().encode('utf-8') - size = Int32.encode(len(client_first)) - self._send_bytes_blocking(size + client_first) - - (data_len,) = struct.unpack('>i', self._recv_bytes_blocking(4)) - server_first = self._recv_bytes_blocking(data_len).decode('utf-8') - scram_client.process_server_first_message(server_first) - - client_final = scram_client.final_message().encode('utf-8') - size = Int32.encode(len(client_final)) - self._send_bytes_blocking(size + client_final) - - (data_len,) = struct.unpack('>i', self._recv_bytes_blocking(4)) - server_final = self._recv_bytes_blocking(data_len).decode('utf-8') - scram_client.process_server_final_message(server_final) + try: + data = self._recv_bytes_blocking(4) + nbytes = Int32.decode(io.BytesIO(data)) + data += self._recv_bytes_blocking(nbytes) + except (ConnectionError, TimeoutError) as e: + log.exception("%s: Error receiving sasl auth bytes from server", self) + err = Errors.KafkaConnectionError("%s: %s" % (self, e)) + self.close(error=err) + return - except (ConnectionError, TimeoutError) as e: - log.exception("%s: Error receiving reply from server", self) - err = Errors.KafkaConnectionError("%s: %s" % (self, e)) - close = True + if version == 1: + ((correlation_id, response),) = self._protocol.receive_bytes(data) + (future, timestamp, _timeout) = self.in_flight_requests.pop(correlation_id) + latency_ms = (time.time() - timestamp) * 1000 + if self._sensors: + self._sensors.request_time.record(latency_ms) + log.debug('%s Response %d (%s ms): %s', self, correlation_id, latency_ms, response) - if err is not None: - if close: - self.close(error=err) - return future.failure(err) - - log.info( - '%s: Authenticated as %s via %s', self, self.config['sasl_plain_username'], self.config['sasl_mechanism'] - ) - return future.success(True) - - def _try_authenticate_gssapi(self, future): - kerberos_damin_name = self.config['sasl_kerberos_domain_name'] or self.host - auth_id = self.config['sasl_kerberos_service_name'] + '@' + kerberos_damin_name - gssapi_name = gssapi.Name( - auth_id, - name_type=gssapi.NameType.hostbased_service - ).canonicalize(gssapi.MechType.kerberos) - log.debug('%s: GSSAPI name: %s', self, gssapi_name) + error_type = Errors.for_code(response.error_code) + if error_type is not Errors.NoError: + log.error("%s: SaslAuthenticate error: %s (%s)", + self, error_type.__name__, response.error_message) + self.close(error=error_type(response.error_message)) + return + return response.auth_bytes + else: + # unframed bytes w/ SaslHandhake v0 + return data[4:] - err = None - close = False - with self._lock: + def _sasl_authenticate(self, future): + while not self._sasl_mechanism.is_done(): + send_token = self._sasl_mechanism.auth_bytes() + self._send_sasl_authenticate(send_token) if not self._can_send_recv(): - err = Errors.NodeNotReadyError(str(self)) - close = False - else: - # Establish security context and negotiate protection level - # For reference RFC 2222, section 7.2.1 - try: - # Exchange tokens until authentication either succeeds or fails - client_ctx = gssapi.SecurityContext(name=gssapi_name, usage='initiate') - received_token = None - while not client_ctx.complete: - # calculate an output token from kafka token (or None if first iteration) - output_token = client_ctx.step(received_token) - - # pass output token to kafka, or send empty response if the security - # context is complete (output token is None in that case) - if output_token is None: - self._send_bytes_blocking(Int32.encode(0)) - else: - msg = output_token - size = Int32.encode(len(msg)) - self._send_bytes_blocking(size + msg) - - # The server will send a token back. Processing of this token either - # establishes a security context, or it needs further token exchange. - # The gssapi will be able to identify the needed next step. - # The connection is closed on failure. - header = self._recv_bytes_blocking(4) - (token_size,) = struct.unpack('>i', header) - received_token = self._recv_bytes_blocking(token_size) - - # Process the security layer negotiation token, sent by the server - # once the security context is established. - - # unwraps message containing supported protection levels and msg size - msg = client_ctx.unwrap(received_token).message - # Kafka currently doesn't support integrity or confidentiality security layers, so we - # simply set QoP to 'auth' only (first octet). We reuse the max message size proposed - # by the server - msg = Int8.encode(SASL_QOP_AUTH & Int8.decode(io.BytesIO(msg[0:1]))) + msg[1:] - # add authorization identity to the response, GSS-wrap and send it - msg = client_ctx.wrap(msg + auth_id.encode(), False).message - size = Int32.encode(len(msg)) - self._send_bytes_blocking(size + msg) - - except (ConnectionError, TimeoutError) as e: - log.exception("%s: Error receiving reply from server", self) - err = Errors.KafkaConnectionError("%s: %s" % (self, e)) - close = True - except Exception as e: - err = e - close = True - - if err is not None: - if close: - self.close(error=err) - return future.failure(err) - - log.info('%s: Authenticated as %s via GSSAPI', self, gssapi_name) - return future.success(True) + return future.failure(Errors.KafkaConnectionError("%s: Connection failure during Sasl Authenticate" % self)) - def _try_authenticate_oauth(self, future): - data = b'' - - msg = bytes(self._build_oauth_client_request().encode("utf-8")) - size = Int32.encode(len(msg)) - - err = None - close = False - with self._lock: - if not self._can_send_recv(): - err = Errors.NodeNotReadyError(str(self)) - close = False + recv_token = self._recv_sasl_authenticate() + if recv_token is None: + return future.failure(Errors.KafkaConnectionError("%s: Connection failure during Sasl Authenticate" % self)) else: - try: - # Send SASL OAuthBearer request with OAuth token - self._send_bytes_blocking(size + msg) - - # The server will send a zero sized message (that is Int32(0)) on success. - # The connection is closed on failure - data = self._recv_bytes_blocking(4) + self._sasl_mechanism.receive(recv_token) - except (ConnectionError, TimeoutError) as e: - log.exception("%s: Error receiving reply from server", self) - err = Errors.KafkaConnectionError("%s: %s" % (self, e)) - close = True - - if err is not None: - if close: - self.close(error=err) - return future.failure(err) - - if data != b'\x00\x00\x00\x00': - error = Errors.AuthenticationFailedError('Unrecognized response during authentication') - return future.failure(error) - - log.info('%s: Authenticated via OAuth', self) - return future.success(True) - - def _build_oauth_client_request(self): - token_provider = self.config['sasl_oauth_token_provider'] - return "n,,\x01auth=Bearer {}{}\x01\x01".format(token_provider.token(), self._token_extensions()) - - def _token_extensions(self): - """ - Return a string representation of the OPTIONAL key-value pairs that can be sent with an OAUTHBEARER - initial request. - """ - token_provider = self.config['sasl_oauth_token_provider'] - - # Only run if the #extensions() method is implemented by the clients Token Provider class - # Builds up a string separated by \x01 via a dict of key value pairs - if callable(getattr(token_provider, "extensions", None)) and len(token_provider.extensions()) > 0: - msg = "\x01".join(["{}={}".format(k, v) for k, v in token_provider.extensions().items()]) - return "\x01" + msg + if self._sasl_mechanism.is_authenticated(): + log.info('%s: %s', self, self._sasl_mechanism.auth_details()) + return future.success(True) else: - return "" + return future.failure(Errors.AuthenticationFailedError('Failed to authenticate via SASL %s' % self.config['sasl_mechanism'])) def blacked_out(self): """ @@ -844,20 +777,43 @@ def blacked_out(self): re-establish a connection yet """ if self.state is ConnectionStates.DISCONNECTED: - if time.time() < self.last_attempt + self._reconnect_backoff: - return True + return self.connection_delay() > 0 return False + def throttled(self): + """ + Return True if we are connected but currently throttled. + """ + if self.state is not ConnectionStates.CONNECTED: + return False + return self.throttle_delay() > 0 + + def throttle_delay(self): + """ + Return the number of milliseconds to wait until connection is no longer throttled. + """ + if self._throttle_time is not None: + remaining_ms = (self._throttle_time - time.time()) * 1000 + if remaining_ms > 0: + return remaining_ms + else: + self._throttle_time = None + return 0 + return 0 + def connection_delay(self): """ Return the number of milliseconds to wait, based on the connection - state, before attempting to send data. When disconnected, this respects - the reconnect backoff time. When connecting or connected, returns a very + state, before attempting to send data. When connecting or disconnected, + this respects the reconnect backoff time. When connected, returns a very large number to handle slow/stalled connections. """ - time_waited = time.time() - (self.last_attempt or 0) - if self.state is ConnectionStates.DISCONNECTED: - return max(self._reconnect_backoff - time_waited, 0) * 1000 + if self.disconnected() or self.connecting(): + if len(self._gai) > 0: + return 0 + else: + time_waited = time.time() - self.last_attempt + return max(self._reconnect_backoff - time_waited, 0) * 1000 else: # When connecting or connected, we should be able to delay # indefinitely since other events (connection or data acked) will @@ -873,7 +829,17 @@ def connecting(self): different states, such as SSL handshake, authorization, etc).""" return self.state in (ConnectionStates.CONNECTING, ConnectionStates.HANDSHAKE, - ConnectionStates.AUTHENTICATING) + ConnectionStates.AUTHENTICATING, + ConnectionStates.API_VERSIONS_SEND, + ConnectionStates.API_VERSIONS_RECV) + + def initializing(self): + """Returns True if socket is connected but full connection is not complete. + During this time the connection may send api requests to the broker to + check api versions and perform SASL authentication.""" + return self.state in (ConnectionStates.AUTHENTICATING, + ConnectionStates.API_VERSIONS_SEND, + ConnectionStates.API_VERSIONS_RECV) def disconnected(self): """Return True iff socket is closed""" @@ -883,6 +849,9 @@ def _reset_reconnect_backoff(self): self._failures = 0 self._reconnect_backoff = self.config['reconnect_backoff_ms'] / 1000.0 + def _reconnect_jitter_pct(self): + return uniform(0.8, 1.2) + def _update_reconnect_backoff(self): # Do not mark as failure if there are more dns entries available to try if len(self._gai) > 0: @@ -891,7 +860,7 @@ def _update_reconnect_backoff(self): self._failures += 1 self._reconnect_backoff = self.config['reconnect_backoff_ms'] * 2 ** (self._failures - 1) self._reconnect_backoff = min(self._reconnect_backoff, self.config['reconnect_backoff_max_ms']) - self._reconnect_backoff *= uniform(0.8, 1.2) + self._reconnect_backoff *= self._reconnect_jitter_pct() self._reconnect_backoff /= 1000.0 log.debug('%s: reconnect backoff %s after %s failures', self, self._reconnect_backoff, self._failures) @@ -916,8 +885,9 @@ def close(self, error=None): with self._lock: if self.state is ConnectionStates.DISCONNECTED: return - log.info('%s: Closing connection. %s', self, error or '') + log.log(logging.ERROR if error else logging.INFO, '%s: Closing connection. %s', self, error or '') self._update_reconnect_backoff() + self._api_versions_future = None self._sasl_auth_future = None self._protocol = KafkaProtocol( client_id=self.config['client_id'], @@ -939,26 +909,40 @@ def close(self, error=None): # drop lock before state change callback and processing futures self.config['state_change_callback'](self.node_id, sock, self) sock.close() - for (_correlation_id, (future, _timestamp)) in ifrs: + for (_correlation_id, (future, _timestamp, _timeout)) in ifrs: future.failure(error) def _can_send_recv(self): """Return True iff socket is ready for requests / responses""" - return self.state in (ConnectionStates.AUTHENTICATING, - ConnectionStates.CONNECTED) + return self.connected() or self.initializing() + + def send(self, request, blocking=True, request_timeout_ms=None): + """Queue request for async network send, return Future() + + Arguments: + request (Request): kafka protocol request object to send. + + Keyword Arguments: + blocking (bool, optional): Whether to immediately send via + blocking socket I/O. Default: True. + request_timeout_ms: Custom timeout in milliseconds for request. + Default: None (uses value from connection configuration) - def send(self, request, blocking=True): - """Queue request for async network send, return Future()""" + Returns: future + """ future = Future() if self.connecting(): return future.failure(Errors.NodeNotReadyError(str(self))) elif not self.connected(): return future.failure(Errors.KafkaConnectionError(str(self))) elif not self.can_send_more(): + # very small race here, but prefer it over breaking abstraction to check self._throttle_time + if self.throttled(): + return future.failure(Errors.ThrottlingQuotaExceededError(str(self))) return future.failure(Errors.TooManyInFlightRequests(str(self))) - return self._send(request, blocking=blocking) + return self._send(request, blocking=blocking, request_timeout_ms=request_timeout_ms) - def _send(self, request, blocking=True): + def _send(self, request, blocking=True, request_timeout_ms=None): future = Future() with self._lock: if not self._can_send_recv(): @@ -971,9 +955,11 @@ def _send(self, request, blocking=True): log.debug('%s Request %d: %s', self, correlation_id, request) if request.expect_response(): - sent_time = time.time() assert correlation_id not in self.in_flight_requests, 'Correlation ID already in-flight!' - self.in_flight_requests[correlation_id] = (future, sent_time) + sent_time = time.time() + request_timeout_ms = request_timeout_ms or self.config['request_timeout_ms'] + timeout_at = sent_time + (request_timeout_ms / 1000) + self.in_flight_requests[correlation_id] = (future, sent_time, timeout_at) else: future.success(None) @@ -1040,8 +1026,26 @@ def send_pending_requests_v2(self): self.close(error=error) return False + def _maybe_throttle(self, response): + throttle_time_ms = getattr(response, 'throttle_time_ms', 0) + if self._sensors: + self._sensors.throttle_time.record(throttle_time_ms) + if not throttle_time_ms: + if self._throttle_time is not None: + self._throttle_time = None + return + # Client side throttling enabled in v2.0 brokers + # prior to that throttling (if present) was managed broker-side + if self.config['api_version'] is not None and self.config['api_version'] >= (2, 0): + throttle_time = time.time() + throttle_time_ms / 1000 + self._throttle_time = max(throttle_time, self._throttle_time or 0) + log.warning("%s: %s throttled by broker (%d ms)", self, + response.__class__.__name__, throttle_time_ms) + def can_send_more(self): - """Return True unless there are max_in_flight_requests_per_connection.""" + """Check for throttling / quota violations and max in-flight-requests""" + if self.throttle_delay() > 0: + return False max_ifrs = self.config['max_in_flight_requests_per_connection'] return len(self.in_flight_requests) < max_ifrs @@ -1052,18 +1056,20 @@ def recv(self): """ responses = self._recv() if not responses and self.requests_timed_out(): + timed_out = self.timed_out_ifrs() + timeout_ms = (timed_out[0][2] - timed_out[0][1]) * 1000 log.warning('%s timed out after %s ms. Closing connection.', - self, self.config['request_timeout_ms']) + self, timeout_ms) self.close(error=Errors.RequestTimedOutError( 'Request timed out after %s ms' % - self.config['request_timeout_ms'])) + timeout_ms)) return () # augment responses w/ correlation_id, future, and timestamp for i, (correlation_id, response) in enumerate(responses): try: with self._lock: - (future, timestamp) = self.in_flight_requests.pop(correlation_id) + (future, timestamp, _timeout) = self.in_flight_requests.pop(correlation_id) except KeyError: self.close(Errors.KafkaConnectionError('Received unrecognized correlation id')) return () @@ -1072,6 +1078,7 @@ def recv(self): self._sensors.request_time.record(latency_ms) log.debug('%s Response %d (%s ms): %s', self, correlation_id, latency_ms, response) + self._maybe_throttle(response) responses[i] = (response, future) return responses @@ -1132,36 +1139,30 @@ def _recv(self): return () def requests_timed_out(self): + return self.next_ifr_request_timeout_ms() == 0 + + def timed_out_ifrs(self): + now = time.time() + ifrs = sorted(self.in_flight_requests.values(), reverse=True, key=lambda ifr: ifr[2]) + return list(filter(lambda ifr: ifr[2] <= now, ifrs)) + + def next_ifr_request_timeout_ms(self): with self._lock: if self.in_flight_requests: - get_timestamp = lambda v: v[1] - oldest_at = min(map(get_timestamp, - self.in_flight_requests.values())) - timeout = self.config['request_timeout_ms'] / 1000.0 - if time.time() >= oldest_at + timeout: - return True - return False - - def _handle_api_version_response(self, response): - error_type = Errors.for_code(response.error_code) - assert error_type is Errors.NoError, "API version check failed" - self._api_versions = dict([ - (api_key, (min_version, max_version)) - for api_key, min_version, max_version in response.api_versions - ]) - return self._api_versions + def get_timeout(v): + return v[2] + next_timeout = min(map(get_timeout, + self.in_flight_requests.values())) + return max(0, (next_timeout - time.time()) * 1000) + else: + return float('inf') def get_api_versions(self): - if self._api_versions is not None: - return self._api_versions - - version = self.check_version() - if version < (0, 10, 0): - raise Errors.UnsupportedVersionError( - "ApiVersion not supported by cluster version {} < 0.10.0" - .format(version)) - # _api_versions is set as a side effect of check_versions() on a cluster - # that supports 0.10.0 or later + # _api_versions is set as a side effect of first connection + # which should typically be bootstrap, but call check_version + # if that hasn't happened yet + if self._api_versions is None: + self.check_version() return self._api_versions def _infer_broker_version_from_api_versions(self, api_versions): @@ -1169,140 +1170,69 @@ def _infer_broker_version_from_api_versions(self, api_versions): # in reverse order. As soon as we find one that works, return it test_cases = [ # format (, ) - ((2, 6, 0), DescribeClientQuotasRequest[0]), - ((2, 5, 0), DescribeAclsRequest_v2), - ((2, 4, 0), ProduceRequest[8]), - ((2, 3, 0), FetchRequest[11]), - ((2, 2, 0), OffsetRequest[5]), - ((2, 1, 0), FetchRequest[10]), - ((2, 0, 0), FetchRequest[8]), - ((1, 1, 0), FetchRequest[7]), - ((1, 0, 0), MetadataRequest[5]), - ((0, 11, 0), MetadataRequest[4]), + # Make sure to update consumer_integration test check when adding newer versions. + # ((3, 9), FetchRequest[17]), + # ((3, 8), ProduceRequest[11]), + # ((3, 7), FetchRequest[16]), + # ((3, 6), AddPartitionsToTxnRequest[4]), + # ((3, 5), FetchRequest[15]), + # ((3, 4), StopReplicaRequest[3]), # broker-internal api... + # ((3, 3), DescribeAclsRequest[3]), + # ((3, 2), JoinGroupRequest[9]), + # ((3, 1), FetchRequest[13]), + # ((3, 0), ListOffsetsRequest[7]), + # ((2, 8), ProduceRequest[9]), + # ((2, 7), FetchRequest[12]), + # ((2, 6), ListGroupsRequest[4]), + # ((2, 5), JoinGroupRequest[7]), + ((2, 6), DescribeClientQuotasRequest[0]), + ((2, 5), DescribeAclsRequest[2]), + ((2, 4), ProduceRequest[8]), + ((2, 3), FetchRequest[11]), + ((2, 2), ListOffsetsRequest[5]), + ((2, 1), FetchRequest[10]), + ((2, 0), FetchRequest[8]), + ((1, 1), FetchRequest[7]), + ((1, 0), MetadataRequest[5]), + ((0, 11), MetadataRequest[4]), ((0, 10, 2), OffsetFetchRequest[2]), ((0, 10, 1), MetadataRequest[2]), ] # Get the best match of test cases - for broker_version, struct in sorted(test_cases, reverse=True): - if struct.API_KEY not in api_versions: + for broker_version, proto_struct in sorted(test_cases, reverse=True): + if proto_struct.API_KEY not in api_versions: continue - min_version, max_version = api_versions[struct.API_KEY] - if min_version <= struct.API_VERSION <= max_version: + min_version, max_version = api_versions[proto_struct.API_KEY] + if min_version <= proto_struct.API_VERSION <= max_version: return broker_version - # We know that ApiVersionResponse is only supported in 0.10+ + # We know that ApiVersionsResponse is only supported in 0.10+ # so if all else fails, choose that return (0, 10, 0) - def check_version(self, timeout=2, strict=False, topics=[]): + def check_version(self, timeout=2, **kwargs): """Attempt to guess the broker version. + Keyword Arguments: + timeout (numeric, optional): Maximum number of seconds to block attempting + to connect and check version. Default 2 + Note: This is a blocking call. - Returns: version tuple, i.e. (0, 10), (0, 9), (0, 8, 2), ... + Returns: version tuple, i.e. (3, 9), (2, 4), etc ... + + Raises: NodeNotReadyError on timeout """ timeout_at = time.time() + timeout - log.info('Probing node %s broker version', self.node_id) - # Monkeypatch some connection configurations to avoid timeouts - override_config = { - 'request_timeout_ms': timeout * 1000, - 'max_in_flight_requests_per_connection': 5 - } - stashed = {} - for key in override_config: - stashed[key] = self.config[key] - self.config[key] = override_config[key] - - def reset_override_configs(): - for key in stashed: - self.config[key] = stashed[key] - - # kafka kills the connection when it doesn't recognize an API request - # so we can send a test request and then follow immediately with a - # vanilla MetadataRequest. If the server did not recognize the first - # request, both will be failed with a ConnectionError that wraps - # socket.error (32, 54, or 104) - from kafka.protocol.admin import ApiVersionRequest, ListGroupsRequest - from kafka.protocol.commit import OffsetFetchRequest, GroupCoordinatorRequest - - test_cases = [ - # All cases starting from 0.10 will be based on ApiVersionResponse - ((0, 10), ApiVersionRequest[0]()), - ((0, 9), ListGroupsRequest[0]()), - ((0, 8, 2), GroupCoordinatorRequest[0]('kafka-python-default-group')), - ((0, 8, 1), OffsetFetchRequest[0]('kafka-python-default-group', [])), - ((0, 8, 0), MetadataRequest[0](topics)), - ] - - for version, request in test_cases: - if not self.connect_blocking(timeout_at - time.time()): - reset_override_configs() - raise Errors.NodeNotReadyError() - f = self.send(request) - # HACK: sleeping to wait for socket to send bytes - time.sleep(0.1) - # when broker receives an unrecognized request API - # it abruptly closes our socket. - # so we attempt to send a second request immediately - # that we believe it will definitely recognize (metadata) - # the attempt to write to a disconnected socket should - # immediately fail and allow us to infer that the prior - # request was unrecognized - mr = self.send(MetadataRequest[0](topics)) - - selector = self.config['selector']() - selector.register(self._sock, selectors.EVENT_READ) - while not (f.is_done and mr.is_done): - selector.select(1) - for response, future in self.recv(): - future.success(response) - selector.close() - - if f.succeeded(): - if isinstance(request, ApiVersionRequest[0]): - # Starting from 0.10 kafka broker we determine version - # by looking at ApiVersionResponse - api_versions = self._handle_api_version_response(f.value) - version = self._infer_broker_version_from_api_versions(api_versions) - log.info('Broker version identified as %s', '.'.join(map(str, version))) - log.info('Set configuration api_version=%s to skip auto' - ' check_version requests on startup', version) - break - - # Only enable strict checking to verify that we understand failure - # modes. For most users, the fact that the request failed should be - # enough to rule out a particular broker version. - if strict: - # If the socket flush hack did not work (which should force the - # connection to close and fail all pending requests), then we - # get a basic Request Timeout. This is not ideal, but we'll deal - if isinstance(f.exception, Errors.RequestTimedOutError): - pass - - # 0.9 brokers do not close the socket on unrecognized api - # requests (bug...). In this case we expect to see a correlation - # id mismatch - elif (isinstance(f.exception, Errors.CorrelationIdError) and - version == (0, 10)): - pass - elif six.PY2: - assert isinstance(f.exception.args[0], socket.error) - assert f.exception.args[0].errno in (32, 54, 104) - else: - assert isinstance(f.exception.args[0], ConnectionError) - log.info("Broker is not v%s -- it did not recognize %s", - version, request.__class__.__name__) + if not self.connect_blocking(timeout_at - time.time()): + raise Errors.NodeNotReadyError() else: - reset_override_configs() - raise Errors.UnrecognizedBrokerVersion() - - reset_override_configs() - return version + return self._api_version def __str__(self): - return "" % ( - self.node_id, self.host, self.port, self.state, + return "" % ( + self.config['client_id'], self.node_id, self.host, self.port, self.state, AFI_NAMES[self._sock_afi], self._sock_addr) @@ -1359,6 +1289,16 @@ def __init__(self, metrics, metric_group_prefix, node_id): 'The maximum request latency in ms.'), Max()) + throttle_time = metrics.sensor('throttle-time') + throttle_time.add(metrics.metric_name( + 'throttle-time-avg', metric_group_name, + 'The average throttle time in ms.'), + Avg()) + throttle_time.add(metrics.metric_name( + 'throttle-time-max', metric_group_name, + 'The maximum throttle time in ms.'), + Max()) + # if one sensor of the metrics has been registered for the connection, # then all other sensors should have been registered; and vice versa node_str = 'node-{0}'.format(node_id) @@ -1410,9 +1350,23 @@ def __init__(self, metrics, metric_group_prefix, node_id): 'The maximum request latency in ms.'), Max()) + throttle_time = metrics.sensor( + node_str + '.throttle', + parents=[metrics.get_sensor('throttle-time')]) + throttle_time.add(metrics.metric_name( + 'throttle-time-avg', metric_group_name, + 'The average throttle time in ms.'), + Avg()) + throttle_time.add(metrics.metric_name( + 'throttle-time-max', metric_group_name, + 'The maximum throttle time in ms.'), + Max()) + + self.bytes_sent = metrics.sensor(node_str + '.bytes-sent') self.bytes_received = metrics.sensor(node_str + '.bytes-received') self.request_time = metrics.sensor(node_str + '.latency') + self.throttle_time = metrics.sensor(node_str + '.throttle') def _address_family(address): diff --git a/kafka/consumer/fetcher.py b/kafka/consumer/fetcher.py index 7ff9daf7b..a833a5b79 100644 --- a/kafka/consumer/fetcher.py +++ b/kafka/consumer/fetcher.py @@ -13,12 +13,12 @@ from kafka.future import Future from kafka.metrics.stats import Avg, Count, Max, Rate from kafka.protocol.fetch import FetchRequest -from kafka.protocol.offset import ( - OffsetRequest, OffsetResetStrategy, UNKNOWN_OFFSET +from kafka.protocol.list_offsets import ( + ListOffsetsRequest, OffsetResetStrategy, UNKNOWN_OFFSET ) from kafka.record import MemoryRecords from kafka.serializer import Deserializer -from kafka.structs import TopicPartition, OffsetAndTimestamp +from kafka.structs import TopicPartition, OffsetAndMetadata, OffsetAndTimestamp log = logging.getLogger(__name__) @@ -28,7 +28,7 @@ READ_COMMITTED = 1 ConsumerRecord = collections.namedtuple("ConsumerRecord", - ["topic", "partition", "offset", "timestamp", "timestamp_type", + ["topic", "partition", "leader_epoch", "offset", "timestamp", "timestamp_type", "key", "value", "headers", "checksum", "serialized_key_size", "serialized_value_size", "serialized_header_size"]) @@ -57,8 +57,8 @@ class Fetcher(six.Iterator): 'check_crcs': True, 'iterator_refetch_records': 1, # undocumented -- interface may change 'metric_group_prefix': 'consumer', - 'api_version': (0, 8, 0), - 'retry_backoff_ms': 100 + 'retry_backoff_ms': 100, + 'enable_incremental_fetch_sessions': True, } def __init__(self, client, subscriptions, metrics, **configs): @@ -69,6 +69,8 @@ def __init__(self, client, subscriptions, metrics, **configs): raw message key and returns a deserialized key. value_deserializer (callable, optional): Any callable that takes a raw message value and returns a deserialized value. + enable_incremental_fetch_sessions: (bool): Use incremental fetch sessions + when available / supported by kafka broker. See KIP-227. Default: True. fetch_min_bytes (int): Minimum amount of data the server should return for a fetch request, otherwise wait up to fetch_max_wait_ms for more data to accumulate. Default: 1. @@ -111,6 +113,7 @@ def __init__(self, client, subscriptions, metrics, **configs): self._fetch_futures = collections.deque() self._sensors = FetchManagerMetrics(metrics, self.config['metric_group_prefix']) self._isolation_level = READ_UNCOMMITTED + self._session_handlers = {} def send_fetches(self): """Send FetchRequests for all assigned partitions that do not already have @@ -120,12 +123,12 @@ def send_fetches(self): List of Futures: each future resolves to a FetchResponse """ futures = [] - for node_id, request in six.iteritems(self._create_fetch_requests()): + for node_id, (request, fetch_offsets) in six.iteritems(self._create_fetch_requests()): if self._client.ready(node_id): log.debug("Sending FetchRequest to node %s", node_id) future = self._client.send(node_id, request, wakeup=False) - future.add_callback(self._handle_fetch_response, request, time.time()) - future.add_errback(log.error, 'Fetch to node %s failed: %s', node_id) + future.add_callback(self._handle_fetch_response, node_id, fetch_offsets, time.time()) + future.add_errback(self._handle_fetch_error, node_id) futures.append(future) self._fetch_futures.extend(futures) self._clean_done_fetch_futures() @@ -195,9 +198,6 @@ def get_offsets_by_times(self, timestamps, timeout_ms): for tp in timestamps: if tp not in offsets: offsets[tp] = None - else: - offset, timestamp = offsets[tp] - offsets[tp] = OffsetAndTimestamp(offset, timestamp) return offsets def beginning_offsets(self, partitions, timeout_ms): @@ -212,7 +212,7 @@ def beginning_or_end_offset(self, partitions, timestamp, timeout_ms): timestamps = dict([(tp, timestamp) for tp in partitions]) offsets = self._retrieve_offsets(timestamps, timeout_ms) for tp in timestamps: - offsets[tp] = offsets[tp][0] + offsets[tp] = offsets[tp].offset return offsets def _reset_offset(self, partition): @@ -237,7 +237,7 @@ def _reset_offset(self, partition): offsets = self._retrieve_offsets({partition: timestamp}) if partition in offsets: - offset = offsets[partition][0] + offset = offsets[partition].offset # we might lose the assignment while fetching the offset, # so check it is still active @@ -258,8 +258,8 @@ def _retrieve_offsets(self, timestamps, timeout_ms=float("inf")): available. Otherwise timestamp is treated as epoch milliseconds. Returns: - {TopicPartition: (int, int)}: Mapping of partition to - retrieved offset and timestamp. If offset does not exist for + {TopicPartition: OffsetAndTimestamp}: Mapping of partition to + retrieved offset, timestamp, and leader_epoch. If offset does not exist for the provided timestamp, that partition will be missing from this mapping. """ @@ -273,7 +273,7 @@ def _retrieve_offsets(self, timestamps, timeout_ms=float("inf")): if not timestamps: return {} - future = self._send_offset_requests(timestamps) + future = self._send_list_offsets_requests(timestamps) self._client.poll(future=future, timeout_ms=remaining_ms) if future.succeeded(): @@ -370,20 +370,22 @@ def _append(self, drained, part, max_records, update_offsets): log.debug("Not returning fetched records for assigned partition" " %s since it is no longer fetchable", tp) - elif fetch_offset == position: + elif fetch_offset == position.offset: # we are ensured to have at least one record since we already checked for emptiness part_records = part.take(max_records) next_offset = part_records[-1].offset + 1 + leader_epoch = part_records[-1].leader_epoch log.log(0, "Returning fetched records at offset %d for assigned" - " partition %s and update position to %s", position, - tp, next_offset) + " partition %s and update position to %s (leader epoch %s)", position.offset, + tp, next_offset, leader_epoch) for record in part_records: drained[tp].append(record) if update_offsets: - self._subscriptions.assignment[tp].position = next_offset + # TODO: save leader_epoch + self._subscriptions.assignment[tp].position = OffsetAndMetadata(next_offset, '', -1) return len(part_records) else: @@ -391,7 +393,7 @@ def _append(self, drained, part, max_records, update_offsets): # position, ignore them they must be from an obsolete request log.debug("Ignoring fetched records for %s at offset %s since" " the current position is %d", tp, part.fetch_offset, - position) + position.offset) part.discard() return 0 @@ -412,10 +414,10 @@ def _message_generator(self): tp = self._next_partition_records.topic_partition - # We can ignore any prior signal to drop pending message sets + # We can ignore any prior signal to drop pending record batches # because we are starting from a fresh one where fetch_offset == position # i.e., the user seek()'d to this position - self._subscriptions.assignment[tp].drop_pending_message_set = False + self._subscriptions.assignment[tp].drop_pending_record_batch = False for msg in self._next_partition_records.take(): @@ -431,37 +433,49 @@ def _message_generator(self): break # If there is a seek during message iteration, - # we should stop unpacking this message set and + # we should stop unpacking this record batch and # wait for a new fetch response that aligns with the # new seek position - elif self._subscriptions.assignment[tp].drop_pending_message_set: - log.debug("Skipping remainder of message set for partition %s", tp) - self._subscriptions.assignment[tp].drop_pending_message_set = False + elif self._subscriptions.assignment[tp].drop_pending_record_batch: + log.debug("Skipping remainder of record batch for partition %s", tp) + self._subscriptions.assignment[tp].drop_pending_record_batch = False self._next_partition_records = None break # Compressed messagesets may include earlier messages - elif msg.offset < self._subscriptions.assignment[tp].position: + elif msg.offset < self._subscriptions.assignment[tp].position.offset: log.debug("Skipping message offset: %s (expecting %s)", msg.offset, - self._subscriptions.assignment[tp].position) + self._subscriptions.assignment[tp].position.offset) continue - self._subscriptions.assignment[tp].position = msg.offset + 1 + self._subscriptions.assignment[tp].position = OffsetAndMetadata(msg.offset + 1, '', -1) yield msg self._next_partition_records = None - def _unpack_message_set(self, tp, records): + def _unpack_records(self, tp, records): try: batch = records.next_batch() while batch is not None: - # LegacyRecordBatch cannot access either base_offset or last_offset_delta + # Try DefaultsRecordBatch / message log format v2 + # base_offset, last_offset_delta, and control batches try: - self._subscriptions.assignment[tp].last_offset_from_message_batch = batch.base_offset + \ - batch.last_offset_delta + batch_offset = batch.base_offset + batch.last_offset_delta + leader_epoch = batch.leader_epoch + self._subscriptions.assignment[tp].last_offset_from_record_batch = batch_offset + # Control batches have a single record indicating whether a transaction + # was aborted or committed. + # When isolation_level is READ_COMMITTED (currently unsupported) + # we should also skip all messages from aborted transactions + # For now we only support READ_UNCOMMITTED and so we ignore the + # abort/commit signal. + if batch.is_control_batch: + batch = records.next_batch() + continue except AttributeError: + leader_epoch = -1 pass for record in batch: @@ -478,7 +492,7 @@ def _unpack_message_set(self, tp, records): len(h_key.encode("utf-8")) + (len(h_val) if h_val is not None else 0) for h_key, h_val in headers) if headers else -1 yield ConsumerRecord( - tp.topic, tp.partition, record.offset, record.timestamp, + tp.topic, tp.partition, leader_epoch, record.offset, record.timestamp, record.timestamp_type, key, value, headers, record.checksum, key_size, value_size, header_size) @@ -487,7 +501,7 @@ def _unpack_message_set(self, tp, records): # If unpacking raises StopIteration, it is erroneously # caught by the generator. We want all exceptions to be raised # back to the user. See Issue 545 - except StopIteration as e: + except StopIteration: log.exception('StopIteration raised unpacking messageset') raise RuntimeError('StopIteration raised unpacking messageset') @@ -510,7 +524,7 @@ def _deserialize(self, f, topic, bytes_): return f.deserialize(topic, bytes_) return f(bytes_) - def _send_offset_requests(self, timestamps): + def _send_list_offsets_requests(self, timestamps): """Fetch offsets for each partition in timestamps dict. This may send request to multiple nodes, based on who is Leader for partition. @@ -535,7 +549,8 @@ def _send_offset_requests(self, timestamps): return Future().failure( Errors.LeaderNotAvailableError(partition)) else: - timestamps_by_node[node_id][partition] = timestamp + leader_epoch = -1 + timestamps_by_node[node_id][partition] = (timestamp, leader_epoch) # Aggregate results until we have all list_offsets_future = Future() @@ -555,24 +570,33 @@ def on_fail(err): list_offsets_future.failure(err) for node_id, timestamps in six.iteritems(timestamps_by_node): - _f = self._send_offset_request(node_id, timestamps) + _f = self._send_list_offsets_request(node_id, timestamps) _f.add_callback(on_success) _f.add_errback(on_fail) return list_offsets_future - def _send_offset_request(self, node_id, timestamps): + def _send_list_offsets_request(self, node_id, timestamps_and_epochs): + version = self._client.api_version(ListOffsetsRequest, max_version=4) by_topic = collections.defaultdict(list) - for tp, timestamp in six.iteritems(timestamps): - if self.config['api_version'] >= (0, 10, 1): + for tp, (timestamp, leader_epoch) in six.iteritems(timestamps_and_epochs): + if version >= 4: + data = (tp.partition, leader_epoch, timestamp) + elif version >= 1: data = (tp.partition, timestamp) else: data = (tp.partition, timestamp, 1) by_topic[tp.topic].append(data) - if self.config['api_version'] >= (0, 10, 1): - request = OffsetRequest[1](-1, list(six.iteritems(by_topic))) + if version <= 1: + request = ListOffsetsRequest[version]( + -1, + list(six.iteritems(by_topic))) else: - request = OffsetRequest[0](-1, list(six.iteritems(by_topic))) + request = ListOffsetsRequest[version]( + -1, + self._isolation_level, + list(six.iteritems(by_topic))) + # Client returns a future that only fails on network issues # so create a separate future and attach a callback to update it @@ -580,16 +604,16 @@ def _send_offset_request(self, node_id, timestamps): future = Future() _f = self._client.send(node_id, request) - _f.add_callback(self._handle_offset_response, future) + _f.add_callback(self._handle_list_offsets_response, future) _f.add_errback(lambda e: future.failure(e)) return future - def _handle_offset_response(self, future, response): - """Callback for the response of the list offset call above. + def _handle_list_offsets_response(self, future, response): + """Callback for the response of the ListOffsets api call Arguments: future (Future): the future to update based on response - response (OffsetResponse): response from the server + response (ListOffsetsResponse): response from the server Raises: AssertionError: if response does not match partition @@ -603,43 +627,45 @@ def _handle_offset_response(self, future, response): if error_type is Errors.NoError: if response.API_VERSION == 0: offsets = partition_info[2] - assert len(offsets) <= 1, 'Expected OffsetResponse with one offset' + assert len(offsets) <= 1, 'Expected ListOffsetsResponse with one offset' if not offsets: offset = UNKNOWN_OFFSET else: offset = offsets[0] - log.debug("Handling v0 ListOffsetResponse response for %s. " - "Fetched offset %s", partition, offset) - if offset != UNKNOWN_OFFSET: - timestamp_offset_map[partition] = (offset, None) - else: + timestamp = None + leader_epoch = -1 + elif response.API_VERSION <= 3: timestamp, offset = partition_info[2:] - log.debug("Handling ListOffsetResponse response for %s. " - "Fetched offset %s, timestamp %s", - partition, offset, timestamp) - if offset != UNKNOWN_OFFSET: - timestamp_offset_map[partition] = (offset, timestamp) + leader_epoch = -1 + else: + timestamp, offset, leader_epoch = partition_info[2:] + log.debug("Handling ListOffsetsResponse response for %s. " + "Fetched offset %s, timestamp %s, leader_epoch %s", + partition, offset, timestamp, leader_epoch) + if offset != UNKNOWN_OFFSET: + timestamp_offset_map[partition] = OffsetAndTimestamp(offset, timestamp, leader_epoch) elif error_type is Errors.UnsupportedForMessageFormatError: # The message format on the broker side is before 0.10.0, # we simply put None in the response. log.debug("Cannot search by timestamp for partition %s because the" " message format version is before 0.10.0", partition) - elif error_type is Errors.NotLeaderForPartitionError: + elif error_type in (Errors.NotLeaderForPartitionError, + Errors.ReplicaNotAvailableError, + Errors.KafkaStorageError): log.debug("Attempt to fetch offsets for partition %s failed due" - " to obsolete leadership information, retrying.", - partition) + " to %s, retrying.", error_type.__name__, partition) future.failure(error_type(partition)) return elif error_type is Errors.UnknownTopicOrPartitionError: - log.warning("Received unknown topic or partition error in ListOffset " - "request for partition %s. The topic/partition " + - "may not exist or the user may not have Describe access " - "to it.", partition) + log.warning("Received unknown topic or partition error in ListOffsets " + "request for partition %s. The topic/partition " + + "may not exist or the user may not have Describe access " + "to it.", partition) future.failure(error_type(partition)) return else: log.warning("Attempt to fetch offsets for partition %s failed due to:" - " %s", partition, error_type) + " %s", partition, error_type.__name__) future.failure(error_type(partition)) return if not future.is_done: @@ -662,24 +688,25 @@ def _create_fetch_requests(self): FetchRequests skipped if no leader, or node has requests in flight Returns: - dict: {node_id: FetchRequest, ...} (version depends on api_version) + dict: {node_id: (FetchRequest, {TopicPartition: fetch_offset}), ...} (version depends on client api_versions) """ # create the fetch info as a dict of lists of partition info tuples # which can be passed to FetchRequest() via .items() - fetchable = collections.defaultdict(lambda: collections.defaultdict(list)) + version = self._client.api_version(FetchRequest, max_version=10) + fetchable = collections.defaultdict(dict) for partition in self._fetchable_partitions(): node_id = self._client.cluster.leader_for_partition(partition) # advance position for any deleted compacted messages if required - if self._subscriptions.assignment[partition].last_offset_from_message_batch: - next_offset_from_batch_header = self._subscriptions.assignment[partition].last_offset_from_message_batch + 1 - if next_offset_from_batch_header > self._subscriptions.assignment[partition].position: + if self._subscriptions.assignment[partition].last_offset_from_record_batch: + next_offset_from_batch_header = self._subscriptions.assignment[partition].last_offset_from_record_batch + 1 + if next_offset_from_batch_header > self._subscriptions.assignment[partition].position.offset: log.debug( - "Advance position for partition %s from %s to %s (last message batch location plus one)" - " to correct for deleted compacted messages", - partition, self._subscriptions.assignment[partition].position, next_offset_from_batch_header) - self._subscriptions.assignment[partition].position = next_offset_from_batch_header + "Advance position for partition %s from %s to %s (last record batch location plus one)" + " to correct for deleted compacted messages and/or transactional control records", + partition, self._subscriptions.assignment[partition].position.offset, next_offset_from_batch_header) + self._subscriptions.assignment[partition].position = OffsetAndMetadata(next_offset_from_batch_header, '', -1) position = self._subscriptions.assignment[partition].position @@ -689,71 +716,101 @@ def _create_fetch_requests(self): " Requesting metadata update", partition) self._client.cluster.request_update() - elif self._client.in_flight_request_count(node_id) == 0: + elif self._client.in_flight_request_count(node_id) > 0: + log.log(0, "Skipping fetch for partition %s because there is an inflight request to node %s", + partition, node_id) + continue + + if version < 5: partition_info = ( partition.partition, - position, + position.offset, self.config['max_partition_fetch_bytes'] ) - fetchable[node_id][partition.topic].append(partition_info) - log.debug("Adding fetch request for partition %s at offset %d", - partition, position) + elif version <= 8: + partition_info = ( + partition.partition, + position.offset, + -1, # log_start_offset is used internally by brokers / replicas only + self.config['max_partition_fetch_bytes'], + ) else: - log.log(0, "Skipping fetch for partition %s because there is an inflight request to node %s", - partition, node_id) + partition_info = ( + partition.partition, + position.leader_epoch, + position.offset, + -1, # log_start_offset is used internally by brokers / replicas only + self.config['max_partition_fetch_bytes'], + ) + + fetchable[node_id][partition] = partition_info + log.debug("Adding fetch request for partition %s at offset %d", + partition, position.offset) - if self.config['api_version'] >= (0, 11, 0): - version = 4 - elif self.config['api_version'] >= (0, 10, 1): - version = 3 - elif self.config['api_version'] >= (0, 10): - version = 2 - elif self.config['api_version'] == (0, 9): - version = 1 - else: - version = 0 requests = {} - for node_id, partition_data in six.iteritems(fetchable): - if version < 3: - requests[node_id] = FetchRequest[version]( + for node_id, next_partitions in six.iteritems(fetchable): + if version >= 7 and self.config['enable_incremental_fetch_sessions']: + if node_id not in self._session_handlers: + self._session_handlers[node_id] = FetchSessionHandler(node_id) + session = self._session_handlers[node_id].build_next(next_partitions) + else: + # No incremental fetch support + session = FetchRequestData(next_partitions, None, FetchMetadata.LEGACY) + + if version <= 2: + request = FetchRequest[version]( + -1, # replica_id + self.config['fetch_max_wait_ms'], + self.config['fetch_min_bytes'], + session.to_send) + elif version == 3: + request = FetchRequest[version]( -1, # replica_id self.config['fetch_max_wait_ms'], self.config['fetch_min_bytes'], - partition_data.items()) + self.config['fetch_max_bytes'], + session.to_send) + elif version <= 6: + request = FetchRequest[version]( + -1, # replica_id + self.config['fetch_max_wait_ms'], + self.config['fetch_min_bytes'], + self.config['fetch_max_bytes'], + self._isolation_level, + session.to_send) else: - # As of version == 3 partitions will be returned in order as - # they are requested, so to avoid starvation with - # `fetch_max_bytes` option we need this shuffle - # NOTE: we do have partition_data in random order due to usage - # of unordered structures like dicts, but that does not - # guarantee equal distribution, and starting in Python3.6 - # dicts retain insert order. - partition_data = list(partition_data.items()) - random.shuffle(partition_data) - if version == 3: - requests[node_id] = FetchRequest[version]( - -1, # replica_id - self.config['fetch_max_wait_ms'], - self.config['fetch_min_bytes'], - self.config['fetch_max_bytes'], - partition_data) + # Through v8 + request = FetchRequest[version]( + -1, # replica_id + self.config['fetch_max_wait_ms'], + self.config['fetch_min_bytes'], + self.config['fetch_max_bytes'], + self._isolation_level, + session.id, + session.epoch, + session.to_send, + session.to_forget) + + fetch_offsets = {} + for tp, partition_data in six.iteritems(next_partitions): + if version <= 8: + offset = partition_data[1] else: - requests[node_id] = FetchRequest[version]( - -1, # replica_id - self.config['fetch_max_wait_ms'], - self.config['fetch_min_bytes'], - self.config['fetch_max_bytes'], - self._isolation_level, - partition_data) + offset = partition_data[2] + fetch_offsets[tp] = offset + + requests[node_id] = (request, fetch_offsets) + return requests - def _handle_fetch_response(self, request, send_time, response): + def _handle_fetch_response(self, node_id, fetch_offsets, send_time, response): """The callback for fetch completion""" - fetch_offsets = {} - for topic, partitions in request.topics: - for partition_data in partitions: - partition, offset = partition_data[:2] - fetch_offsets[TopicPartition(topic, partition)] = offset + if response.API_VERSION >= 7 and self.config['enable_incremental_fetch_sessions']: + if node_id not in self._session_handlers: + log.error("Unable to find fetch session handler for node %s. Ignoring fetch response", node_id) + return + if not self._session_handlers[node_id].handle_response(response): + return partitions = set([TopicPartition(topic, partition_data[0]) for topic, partitions in response.topics @@ -766,18 +823,23 @@ def _handle_fetch_response(self, request, send_time, response): random.shuffle(partitions) for partition_data in partitions: tp = TopicPartition(topic, partition_data[0]) + fetch_offset = fetch_offsets[tp] completed_fetch = CompletedFetch( - tp, fetch_offsets[tp], + tp, fetch_offset, response.API_VERSION, partition_data[1:], metric_aggregator ) self._completed_fetches.append(completed_fetch) - if response.API_VERSION >= 1: - self._sensors.fetch_throttle_time_sensor.record(response.throttle_time_ms) self._sensors.fetch_latency.record((time.time() - send_time) * 1000) + def _handle_fetch_error(self, node_id, exception): + level = logging.INFO if isinstance(exception, Errors.Cancelled) else logging.ERROR + log.log(level, 'Fetch to node %s failed: %s', node_id, exception) + if node_id in self._session_handlers: + self._session_handlers[node_id].handle_error(exception) + def _parse_fetched_data(self, completed_fetch): tp = completed_fetch.topic_partition fetch_offset = completed_fetch.fetched_offset @@ -803,19 +865,19 @@ def _parse_fetched_data(self, completed_fetch): # Note that the *response* may return a messageset that starts # earlier (e.g., compressed messages) or later (e.g., compacted topic) position = self._subscriptions.assignment[tp].position - if position is None or position != fetch_offset: + if position is None or position.offset != fetch_offset: log.debug("Discarding fetch response for partition %s" " since its offset %d does not match the" " expected offset %d", tp, fetch_offset, - position) + position.offset) return None records = MemoryRecords(completed_fetch.partition_data[-1]) if records.has_next(): log.debug("Adding fetched record for partition %s with" " offset %d to buffered record list", tp, - position) - unpacked = list(self._unpack_message_set(tp, records)) + position.offset) + unpacked = list(self._unpack_records(tp, records)) parsed_records = self.PartitionRecords(fetch_offset, tp, unpacked) if unpacked: last_offset = unpacked[-1].offset @@ -839,14 +901,17 @@ def _parse_fetched_data(self, completed_fetch): self._sensors.record_topic_fetch_metrics(tp.topic, num_bytes, records_count) elif error_type in (Errors.NotLeaderForPartitionError, - Errors.UnknownTopicOrPartitionError): + Errors.ReplicaNotAvailableError, + Errors.UnknownTopicOrPartitionError, + Errors.KafkaStorageError): + log.debug("Error fetching partition %s: %s", tp, error_type.__name__) self._client.cluster.request_update() elif error_type is Errors.OffsetOutOfRangeError: position = self._subscriptions.assignment[tp].position - if position is None or position != fetch_offset: + if position is None or position.offset != fetch_offset: log.debug("Discarding stale fetch response for partition %s" " since the fetched offset %d does not match the" - " current offset %d", tp, fetch_offset, position) + " current offset %d", tp, fetch_offset, position.offset) elif self._subscriptions.has_default_offset_reset_policy(): log.info("Fetch offset %s is out of range for topic-partition %s", fetch_offset, tp) self._subscriptions.need_offset_reset(tp) @@ -856,8 +921,10 @@ def _parse_fetched_data(self, completed_fetch): elif error_type is Errors.TopicAuthorizationFailedError: log.warning("Not authorized to read from topic %s.", tp.topic) raise Errors.TopicAuthorizationFailedError(set(tp.topic)) - elif error_type is Errors.UnknownError: - log.warning("Unknown error fetching data for topic-partition %s", tp) + elif error_type.is_retriable: + log.debug("Retriable error fetching partition %s: %s", tp, error_type()) + if error_type.invalid_metadata: + self._client.cluster.request_update() else: raise error_type('Unexpected error while fetching data') @@ -910,6 +977,212 @@ def take(self, n=None): return res +class FetchSessionHandler(object): + """ + FetchSessionHandler maintains the fetch session state for connecting to a broker. + + Using the protocol outlined by KIP-227, clients can create incremental fetch sessions. + These sessions allow the client to fetch information about a set of partition over + and over, without explicitly enumerating all the partitions in the request and the + response. + + FetchSessionHandler tracks the partitions which are in the session. It also + determines which partitions need to be included in each fetch request, and what + the attached fetch session metadata should be for each request. + """ + + def __init__(self, node_id): + self.node_id = node_id + self.next_metadata = FetchMetadata.INITIAL + self.session_partitions = {} + + def build_next(self, next_partitions): + if self.next_metadata.is_full: + log.debug("Built full fetch %s for node %s with %s partition(s).", + self.next_metadata, self.node_id, len(next_partitions)) + self.session_partitions = next_partitions + return FetchRequestData(next_partitions, None, self.next_metadata) + + prev_tps = set(self.session_partitions.keys()) + next_tps = set(next_partitions.keys()) + log.debug("Building incremental partitions from next: %s, previous: %s", next_tps, prev_tps) + added = next_tps - prev_tps + for tp in added: + self.session_partitions[tp] = next_partitions[tp] + removed = prev_tps - next_tps + for tp in removed: + self.session_partitions.pop(tp) + altered = set() + for tp in next_tps & prev_tps: + if next_partitions[tp] != self.session_partitions[tp]: + self.session_partitions[tp] = next_partitions[tp] + altered.add(tp) + + log.debug("Built incremental fetch %s for node %s. Added %s, altered %s, removed %s out of %s", + self.next_metadata, self.node_id, added, altered, removed, self.session_partitions.keys()) + to_send = {tp: next_partitions[tp] for tp in (added | altered)} + return FetchRequestData(to_send, removed, self.next_metadata) + + def handle_response(self, response): + if response.error_code != Errors.NoError.errno: + error_type = Errors.for_code(response.error_code) + log.info("Node %s was unable to process the fetch request with %s: %s.", + self.node_id, self.next_metadata, error_type()) + if error_type is Errors.FetchSessionIdNotFoundError: + self.next_metadata = FetchMetadata.INITIAL + else: + self.next_metadata = self.next_metadata.next_close_existing() + return False + + response_tps = self._response_partitions(response) + session_tps = set(self.session_partitions.keys()) + if self.next_metadata.is_full: + if response_tps != session_tps: + log.info("Node %s sent an invalid full fetch response with extra %s / omitted %s", + self.node_id, response_tps - session_tps, session_tps - response_tps) + self.next_metadata = FetchMetadata.INITIAL + return False + elif response.session_id == FetchMetadata.INVALID_SESSION_ID: + log.debug("Node %s sent a full fetch response with %s partitions", + self.node_id, len(response_tps)) + self.next_metadata = FetchMetadata.INITIAL + return True + elif response.session_id == FetchMetadata.THROTTLED_SESSION_ID: + log.debug("Node %s sent a empty full fetch response due to a quota violation (%s partitions)", + self.node_id, len(response_tps)) + # Keep current metadata + return True + else: + # The server created a new incremental fetch session. + log.debug("Node %s sent a full fetch response that created a new incremental fetch session %s" + " with %s response partitions", + self.node_id, response.session_id, + len(response_tps)) + self.next_metadata = FetchMetadata.new_incremental(response.session_id) + return True + else: + if response_tps - session_tps: + log.info("Node %s sent an invalid incremental fetch response with extra partitions %s", + self.node_id, response_tps - session_tps) + self.next_metadata = self.next_metadata.next_close_existing() + return False + elif response.session_id == FetchMetadata.INVALID_SESSION_ID: + # The incremental fetch session was closed by the server. + log.debug("Node %s sent an incremental fetch response closing session %s" + " with %s response partitions (%s implied)", + self.node_id, self.next_metadata.session_id, + len(response_tps), len(self.session_partitions) - len(response_tps)) + self.next_metadata = FetchMetadata.INITIAL + return True + elif response.session_id == FetchMetadata.THROTTLED_SESSION_ID: + log.debug("Node %s sent a empty incremental fetch response due to a quota violation (%s partitions)", + self.node_id, len(response_tps)) + # Keep current metadata + return True + else: + # The incremental fetch session was continued by the server. + log.debug("Node %s sent an incremental fetch response for session %s" + " with %s response partitions (%s implied)", + self.node_id, response.session_id, + len(response_tps), len(self.session_partitions) - len(response_tps)) + self.next_metadata = self.next_metadata.next_incremental() + return True + + def handle_error(self, _exception): + self.next_metadata = self.next_metadata.next_close_existing() + + def _response_partitions(self, response): + return {TopicPartition(topic, partition_data[0]) + for topic, partitions in response.topics + for partition_data in partitions} + + +class FetchMetadata(object): + __slots__ = ('session_id', 'epoch') + + MAX_EPOCH = 2147483647 + INVALID_SESSION_ID = 0 # used by clients with no session. + THROTTLED_SESSION_ID = -1 # returned with empty response on quota violation + INITIAL_EPOCH = 0 # client wants to create or recreate a session. + FINAL_EPOCH = -1 # client wants to close any existing session, and not create a new one. + + def __init__(self, session_id, epoch): + self.session_id = session_id + self.epoch = epoch + + @property + def is_full(self): + return self.epoch == self.INITIAL_EPOCH or self.epoch == self.FINAL_EPOCH + + @classmethod + def next_epoch(cls, prev_epoch): + if prev_epoch < 0: + return cls.FINAL_EPOCH + elif prev_epoch == cls.MAX_EPOCH: + return 1 + else: + return prev_epoch + 1 + + def next_close_existing(self): + return self.__class__(self.session_id, self.INITIAL_EPOCH) + + @classmethod + def new_incremental(cls, session_id): + return cls(session_id, cls.next_epoch(cls.INITIAL_EPOCH)) + + def next_incremental(self): + return self.__class__(self.session_id, self.next_epoch(self.epoch)) + +FetchMetadata.INITIAL = FetchMetadata(FetchMetadata.INVALID_SESSION_ID, FetchMetadata.INITIAL_EPOCH) +FetchMetadata.LEGACY = FetchMetadata(FetchMetadata.INVALID_SESSION_ID, FetchMetadata.FINAL_EPOCH) + + +class FetchRequestData(object): + __slots__ = ('_to_send', '_to_forget', '_metadata') + + def __init__(self, to_send, to_forget, metadata): + self._to_send = to_send or dict() # {TopicPartition: (partition, ...)} + self._to_forget = to_forget or set() # {TopicPartition} + self._metadata = metadata + + @property + def metadata(self): + return self._metadata + + @property + def id(self): + return self._metadata.session_id + + @property + def epoch(self): + return self._metadata.epoch + + @property + def to_send(self): + # Return as list of [(topic, [(partition, ...), ...]), ...] + # so it an be passed directly to encoder + partition_data = collections.defaultdict(list) + for tp, partition_info in six.iteritems(self._to_send): + partition_data[tp.topic].append(partition_info) + # As of version == 3 partitions will be returned in order as + # they are requested, so to avoid starvation with + # `fetch_max_bytes` option we need this shuffle + # NOTE: we do have partition_data in random order due to usage + # of unordered structures like dicts, but that does not + # guarantee equal distribution, and starting in Python3.6 + # dicts retain insert order. + return random.sample(list(partition_data.items()), k=len(partition_data)) + + @property + def to_forget(self): + # Return as list of [(topic, (partiiton, ...)), ...] + # so it an be passed directly to encoder + partition_data = collections.defaultdict(list) + for tp in self._to_forget: + partition_data[tp.topic].append(tp.partition) + return list(partition_data.items()) + + class FetchResponseMetricAggregator(object): """ Since we parse the message data for each partition from each fetch @@ -970,12 +1243,6 @@ def __init__(self, metrics, prefix): self.records_fetch_lag.add(metrics.metric_name('records-lag-max', self.group_name, 'The maximum lag in terms of number of records for any partition in self window'), Max()) - self.fetch_throttle_time_sensor = metrics.sensor('fetch-throttle-time') - self.fetch_throttle_time_sensor.add(metrics.metric_name('fetch-throttle-time-avg', self.group_name, - 'The average throttle time in ms'), Avg()) - self.fetch_throttle_time_sensor.add(metrics.metric_name('fetch-throttle-time-max', self.group_name, - 'The maximum throttle time in ms'), Max()) - def record_topic_fetch_metrics(self, topic, num_bytes, num_records): # record bytes fetched name = '.'.join(['topic', topic, 'bytes-fetched']) diff --git a/kafka/consumer/group.py b/kafka/consumer/group.py index a1d1dfa37..e58b8518b 100644 --- a/kafka/consumer/group.py +++ b/kafka/consumer/group.py @@ -16,8 +16,8 @@ from kafka.coordinator.assignors.range import RangePartitionAssignor from kafka.coordinator.assignors.roundrobin import RoundRobinPartitionAssignor from kafka.metrics import MetricConfig, Metrics -from kafka.protocol.offset import OffsetResetStrategy -from kafka.structs import TopicPartition +from kafka.protocol.list_offsets import OffsetResetStrategy +from kafka.structs import OffsetAndMetadata, TopicPartition from kafka.version import __version__ log = logging.getLogger(__name__) @@ -60,6 +60,8 @@ class KafkaConsumer(six.Iterator): raw message key and returns a deserialized key. value_deserializer (callable): Any callable that takes a raw message value and returns a deserialized value. + enable_incremental_fetch_sessions: (bool): Use incremental fetch sessions + when available / supported by kafka broker. See KIP-227. Default: True. fetch_min_bytes (int): Minimum amount of data the server should return for a fetch request, otherwise wait up to fetch_max_wait_ms for more data to accumulate. Default: 1. @@ -98,7 +100,7 @@ class KafkaConsumer(six.Iterator): reconnection attempts will continue periodically with this fixed rate. To avoid connection storms, a randomization factor of 0.2 will be applied to the backoff resulting in a random range between - 20% below and 20% above the computed value. Default: 1000. + 20% below and 20% above the computed value. Default: 30000. max_in_flight_requests_per_connection (int): Requests are pipelined to kafka brokers up to this number of maximum requests per broker connection. Default: 5. @@ -118,6 +120,9 @@ class KafkaConsumer(six.Iterator): consumed. This ensures no on-the-wire or on-disk corruption to the messages occurred. This check adds some overhead, so it may be disabled in cases seeking extreme performance. Default: True + allow_auto_create_topics (bool): Enable/disable auto topic creation + on metadata request. Only available with api_version >= (0, 11). + Default: True metadata_max_age_ms (int): The period of time in milliseconds after which we force a refresh of metadata, even if we haven't seen any partition leadership changes to proactively discover any new @@ -195,10 +200,17 @@ class KafkaConsumer(six.Iterator): or other configuration forbids use of all the specified ciphers), an ssl.SSLError will be raised. See ssl.SSLContext.set_ciphers api_version (tuple): Specify which Kafka API version to use. If set to - None, the client will attempt to infer the broker version by probing - various APIs. Different versions enable different functionality. + None, the client will attempt to determine the broker version via + ApiVersionsRequest API or, for brokers earlier than 0.10, probing + various known APIs. Dynamic version checking is performed eagerly + during __init__ and can raise NoBrokersAvailableError if no connection + was made before timeout (see api_version_auto_timeout_ms below). + Different versions enable different functionality. Examples: + (3, 9) most recent broker release, enable all supported features + (0, 11) enables message format v2 (internal) + (0, 10, 0) enables sasl authentication and message format v1 (0, 9) enables full group coordination features with automatic partition assignment and rebalancing, (0, 8, 2) enables kafka-storage offset commits with manual @@ -212,6 +224,7 @@ class KafkaConsumer(six.Iterator): api_version_auto_timeout_ms (int): number of milliseconds to throw a timeout exception from the constructor when checking the broker api version. Only applies if api_version set to None. + Default: 2000 connections_max_idle_ms: Close idle connections after the number of milliseconds specified by this config. The broker closes idle connections after connections.max.idle.ms, so this avoids hitting @@ -224,6 +237,9 @@ class KafkaConsumer(six.Iterator): metrics. Default: 2 metrics_sample_window_ms (int): The maximum age in milliseconds of samples used to compute metrics. Default: 30000 + coordinator_not_ready_retry_timeout_ms (int): The timeout used to detect + that the Kafka coordinator is not available. If 'None', the default + behavior of polling indefinitely would be kept. Default: None selector (selectors.BaseSelector): Provide a specific selector implementation to use for I/O multiplexing. Default: selectors.DefaultSelector @@ -238,6 +254,9 @@ class KafkaConsumer(six.Iterator): Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms. sasl_plain_password (str): password for sasl PLAIN and SCRAM authentication. Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms. + sasl_kerberos_name (str or gssapi.Name): Constructed gssapi.Name for use with + sasl mechanism handshake. If provided, sasl_kerberos_service_name and + sasl_kerberos_domain name are ignored. Default: None. sasl_kerberos_service_name (str): Service name to include in GSSAPI sasl mechanism handshake. Default: 'kafka' sasl_kerberos_domain_name (str): kerberos domain name to use in GSSAPI @@ -256,6 +275,7 @@ class KafkaConsumer(six.Iterator): 'group_id': None, 'key_deserializer': None, 'value_deserializer': None, + 'enable_incremental_fetch_sessions': True, 'fetch_max_wait_ms': 500, 'fetch_min_bytes': 1, 'fetch_max_bytes': 52428800, @@ -263,13 +283,14 @@ class KafkaConsumer(six.Iterator): 'request_timeout_ms': 305000, # chosen to be higher than the default of max_poll_interval_ms 'retry_backoff_ms': 100, 'reconnect_backoff_ms': 50, - 'reconnect_backoff_max_ms': 1000, + 'reconnect_backoff_max_ms': 30000, 'max_in_flight_requests_per_connection': 5, 'auto_offset_reset': 'latest', 'enable_auto_commit': True, 'auto_commit_interval_ms': 5000, 'default_offset_commit_callback': lambda offsets, response: True, 'check_crcs': True, + 'allow_auto_create_topics': True, 'metadata_max_age_ms': 5 * 60 * 1000, 'partition_assignment_strategy': (RangePartitionAssignor, RoundRobinPartitionAssignor), 'max_poll_records': 500, @@ -298,11 +319,13 @@ class KafkaConsumer(six.Iterator): 'metrics_num_samples': 2, 'metrics_sample_window_ms': 30000, 'metric_group_prefix': 'consumer', + 'coordinator_not_ready_retry_timeout_ms': None, 'selector': selectors.DefaultSelector, 'exclude_internal_topics': True, 'sasl_mechanism': None, 'sasl_plain_username': None, 'sasl_plain_password': None, + 'sasl_kerberos_name': None, 'sasl_kerberos_service_name': 'kafka', 'sasl_kerberos_domain_name': None, 'sasl_oauth_token_provider': None, @@ -357,9 +380,8 @@ def __init__(self, *topics, **configs): self._client = self.config['kafka_client'](metrics=self._metrics, **self.config) - # Get auto-discovered version from client if necessary - if self.config['api_version'] is None: - self.config['api_version'] = self._client.config['api_version'] + # Get auto-discovered / normalized version from client + self.config['api_version'] = self._client.config['api_version'] # Coordinator configurations are different for older brokers # max_poll_interval_ms is not supported directly -- it must the be @@ -719,16 +741,16 @@ def position(self, partition): partition (TopicPartition): Partition to check Returns: - int: Offset + int: Offset or None """ if not isinstance(partition, TopicPartition): raise TypeError('partition must be a TopicPartition namedtuple') assert self._subscription.is_assigned(partition), 'Partition is not assigned' - offset = self._subscription.assignment[partition].position - if offset is None: + position = self._subscription.assignment[partition].position + if position is None: self._update_fetch_positions([partition]) - offset = self._subscription.assignment[partition].position - return offset + position = self._subscription.assignment[partition].position + return position.offset if position else None def highwater(self, partition): """Last known highwater offset for a partition. @@ -1131,7 +1153,7 @@ def _message_generator_v2(self): log.debug("Not returning fetched records for partition %s" " since it is no longer fetchable", tp) break - self._subscription.assignment[tp].position = record.offset + 1 + self._subscription.assignment[tp].position = OffsetAndMetadata(record.offset + 1, '', -1) yield record def _message_generator(self): diff --git a/kafka/consumer/subscription_state.py b/kafka/consumer/subscription_state.py index 08842d133..b30922b3e 100644 --- a/kafka/consumer/subscription_state.py +++ b/kafka/consumer/subscription_state.py @@ -7,7 +7,7 @@ from kafka.vendor import six from kafka.errors import IllegalStateError -from kafka.protocol.offset import OffsetResetStrategy +from kafka.protocol.list_offsets import OffsetResetStrategy from kafka.structs import OffsetAndMetadata log = logging.getLogger(__name__) @@ -319,7 +319,7 @@ def all_consumed_offsets(self): all_consumed = {} for partition, state in six.iteritems(self.assignment): if state.has_valid_position: - all_consumed[partition] = OffsetAndMetadata(state.position, '') + all_consumed[partition] = state.position return all_consumed def need_offset_reset(self, partition, offset_reset_strategy=None): @@ -379,15 +379,16 @@ def __init__(self): self.paused = False # whether this partition has been paused by the user self.awaiting_reset = False # whether we are awaiting reset self.reset_strategy = None # the reset strategy if awaitingReset is set - self._position = None # offset exposed to the user + self._position = None # OffsetAndMetadata exposed to the user self.highwater = None - self.drop_pending_message_set = False - # The last message offset hint available from a message batch with + self.drop_pending_record_batch = False + # The last message offset hint available from a record batch with # magic=2 which includes deleted compacted messages - self.last_offset_from_message_batch = None + self.last_offset_from_record_batch = None def _set_position(self, offset): assert self.has_valid_position, 'Valid position required' + assert isinstance(offset, OffsetAndMetadata) self._position = offset def _get_position(self): @@ -399,16 +400,16 @@ def await_reset(self, strategy): self.awaiting_reset = True self.reset_strategy = strategy self._position = None - self.last_offset_from_message_batch = None + self.last_offset_from_record_batch = None self.has_valid_position = False def seek(self, offset): - self._position = offset + self._position = OffsetAndMetadata(offset, '', -1) self.awaiting_reset = False self.reset_strategy = None self.has_valid_position = True - self.drop_pending_message_set = True - self.last_offset_from_message_batch = None + self.drop_pending_record_batch = True + self.last_offset_from_record_batch = None def pause(self): self.paused = True diff --git a/kafka/coordinator/assignors/sticky/sticky_assignor.py b/kafka/coordinator/assignors/sticky/sticky_assignor.py index dce714f1a..6e79c597e 100644 --- a/kafka/coordinator/assignors/sticky/sticky_assignor.py +++ b/kafka/coordinator/assignors/sticky/sticky_assignor.py @@ -2,7 +2,6 @@ from collections import defaultdict, namedtuple from copy import deepcopy -from kafka.cluster import ClusterMetadata from kafka.coordinator.assignors.abstract import AbstractPartitionAssignor from kafka.coordinator.assignors.sticky.partition_movements import PartitionMovements from kafka.coordinator.assignors.sticky.sorted_set import SortedSet diff --git a/kafka/coordinator/base.py b/kafka/coordinator/base.py index e71984108..5ea12cfa0 100644 --- a/kafka/coordinator/base.py +++ b/kafka/coordinator/base.py @@ -14,9 +14,8 @@ from kafka.future import Future from kafka.metrics import AnonMeasurable from kafka.metrics.stats import Avg, Count, Max, Rate -from kafka.protocol.commit import GroupCoordinatorRequest, OffsetCommitRequest -from kafka.protocol.group import (HeartbeatRequest, JoinGroupRequest, - LeaveGroupRequest, SyncGroupRequest) +from kafka.protocol.find_coordinator import FindCoordinatorRequest +from kafka.protocol.group import HeartbeatRequest, JoinGroupRequest, LeaveGroupRequest, SyncGroupRequest, DEFAULT_GENERATION_ID, UNKNOWN_MEMBER_ID log = logging.getLogger('kafka.coordinator') @@ -33,10 +32,7 @@ def __init__(self, generation_id, member_id, protocol): self.member_id = member_id self.protocol = protocol -Generation.NO_GENERATION = Generation( - OffsetCommitRequest[2].DEFAULT_GENERATION_ID, - JoinGroupRequest[0].UNKNOWN_MEMBER_ID, - None) +Generation.NO_GENERATION = Generation(DEFAULT_GENERATION_ID, UNKNOWN_MEMBER_ID, None) class UnjoinedGroupException(Errors.KafkaError): @@ -84,6 +80,7 @@ class BaseCoordinator(object): 'group_id': 'kafka-python-default-group', 'session_timeout_ms': 10000, 'heartbeat_interval_ms': 3000, + 'coordinator_not_ready_retry_timeout_ms': None, 'max_poll_interval_ms': 300000, 'retry_backoff_ms': 100, 'api_version': (0, 10, 1), @@ -107,6 +104,10 @@ def __init__(self, client, metrics, **configs): should be set no higher than 1/3 of that value. It can be adjusted even lower to control the expected time for normal rebalances. Default: 3000 + coordinator_not_ready_retry_timeout_ms (int): The timeout used to + detect that the Kafka coordinator is not available. If 'None', + the default behavior of polling indefinitely would be kept. + Default: None retry_backoff_ms (int): Milliseconds to backoff when retrying on errors. Default: 100. """ @@ -238,10 +239,12 @@ def coordinator(self): else: return self.coordinator_id - def ensure_coordinator_ready(self): + def ensure_coordinator_ready(self, timeout_ms=None): """Block until the coordinator for this group is known (and we have an active connection -- java client uses unsent queue). """ + retry_timeout_ms = timeout_ms or self.config['coordinator_not_ready_retry_timeout_ms'] + retry_start_time_in_secs = time.time() with self._client._lock, self._lock: while self.coordinator_unknown(): @@ -259,7 +262,15 @@ def ensure_coordinator_ready(self): if future.failed(): if future.retriable(): - if getattr(future.exception, 'invalid_metadata', False): + if retry_timeout_ms is not None and isinstance( + future.exception, (Errors.NodeNotReadyError, Errors.NoBrokersAvailable)): + remaining_retry_timeout_ms = retry_timeout_ms - ( + time.time() - retry_start_time_in_secs) * 1000 + if remaining_retry_timeout_ms <= 0: + raise future.exception # pylint: disable-msg=raising-bad-type + self._client.poll(timeout_ms=min( + self.config['retry_backoff_ms'], remaining_retry_timeout_ms)) + elif getattr(future.exception, 'invalid_metadata', False): log.debug('Requesting metadata for group coordinator request: %s', future.exception) metadata_update = self._client.cluster.request_update() self._client.poll(future=metadata_update) @@ -371,7 +382,7 @@ def ensure_active_group(self): while not self.coordinator_unknown(): if not self._client.in_flight_request_count(self.coordinator_id): break - self._client.poll() + self._client.poll(timeout_ms=200) else: continue @@ -452,25 +463,16 @@ def _send_join_group_request(self): (protocol, metadata if isinstance(metadata, bytes) else metadata.encode()) for protocol, metadata in self.group_protocols() ] - if self.config['api_version'] < (0, 9): - raise Errors.KafkaError('JoinGroupRequest api requires 0.9+ brokers') - elif (0, 9) <= self.config['api_version'] < (0, 10, 1): - request = JoinGroupRequest[0]( + version = self._client.api_version(JoinGroupRequest, max_version=3) + if version == 0: + request = JoinGroupRequest[version]( self.group_id, self.config['session_timeout_ms'], self._generation.member_id, self.protocol_type(), member_metadata) - elif (0, 10, 1) <= self.config['api_version'] < (0, 11, 0): - request = JoinGroupRequest[1]( - self.group_id, - self.config['session_timeout_ms'], - self.config['max_poll_interval_ms'], - self._generation.member_id, - self.protocol_type(), - member_metadata) else: - request = JoinGroupRequest[2]( + request = JoinGroupRequest[version]( self.group_id, self.config['session_timeout_ms'], self.config['max_poll_interval_ms'], @@ -562,7 +564,7 @@ def _handle_join_group_response(self, future, send_time, response): def _on_join_follower(self): # send follower's sync group with an empty assignment - version = 0 if self.config['api_version'] < (0, 11, 0) else 1 + version = self._client.api_version(SyncGroupRequest, max_version=2) request = SyncGroupRequest[version]( self.group_id, self._generation.generation_id, @@ -590,7 +592,7 @@ def _on_join_leader(self, response): except Exception as e: return Future().failure(e) - version = 0 if self.config['api_version'] < (0, 11, 0) else 1 + version = self._client.api_version(SyncGroupRequest, max_version=2) request = SyncGroupRequest[version]( self.group_id, self._generation.generation_id, @@ -669,7 +671,11 @@ def _send_group_coordinator_request(self): log.debug("Sending group coordinator request for group %s to broker %s", self.group_id, node_id) - request = GroupCoordinatorRequest[0](self.group_id) + version = self._client.api_version(FindCoordinatorRequest, max_version=2) + if version == 0: + request = FindCoordinatorRequest[version](self.group_id) + else: + request = FindCoordinatorRequest[version](self.group_id, 0) future = Future() _f = self._client.send(node_id, request) _f.add_callback(self._handle_group_coordinator_response, future) @@ -744,7 +750,7 @@ def _start_heartbeat_thread(self): self._heartbeat_thread.start() def _close_heartbeat_thread(self): - if self._heartbeat_thread is not None: + if hasattr(self, '_heartbeat_thread') and self._heartbeat_thread is not None: log.info('Stopping heartbeat thread') try: self._heartbeat_thread.close() @@ -771,7 +777,7 @@ def maybe_leave_group(self): # this is a minimal effort attempt to leave the group. we do not # attempt any resending if the request fails or times out. log.info('Leaving consumer group (%s).', self.group_id) - version = 0 if self.config['api_version'] < (0, 11, 0) else 1 + version = self._client.api_version(LeaveGroupRequest, max_version=2) request = LeaveGroupRequest[version](self.group_id, self._generation.member_id) future = self._client.send(self.coordinator_id, request) future.add_callback(self._handle_leave_group_response) @@ -799,7 +805,7 @@ def _send_heartbeat_request(self): e = Errors.NodeNotReadyError(self.coordinator_id) return Future().failure(e) - version = 0 if self.config['api_version'] < (0, 11, 0) else 1 + version = self._client.api_version(HeartbeatRequest, max_version=2) request = HeartbeatRequest[version](self.group_id, self._generation.generation_id, self._generation.member_id) @@ -920,9 +926,19 @@ def disable(self): self.enabled = False def close(self): + if self.closed: + return self.closed = True + + # Generally this should not happen - close() is triggered + # by the coordinator. But in some cases GC may close the coordinator + # from within the heartbeat thread. + if threading.current_thread() == self: + return + with self.coordinator._lock: self.coordinator._lock.notify() + if self.is_alive(): self.join(self.coordinator.config['heartbeat_interval_ms'] / 1000) if self.is_alive(): @@ -990,6 +1006,11 @@ def _run_once(self): # foreground thread has stalled in between calls to # poll(), so we explicitly leave the group. log.warning('Heartbeat poll expired, leaving group') + ### XXX + # maybe_leave_group acquires client + coordinator lock; + # if we hold coordinator lock before calling, we risk deadlock + # release() is safe here because this is the last code in the current context + self.coordinator._lock.release() self.coordinator.maybe_leave_group() elif not self.coordinator.heartbeat.should_heartbeat(): diff --git a/kafka/coordinator/consumer.py b/kafka/coordinator/consumer.py index 971f5e802..c1e5d828a 100644 --- a/kafka/coordinator/consumer.py +++ b/kafka/coordinator/consumer.py @@ -39,7 +39,8 @@ class ConsumerCoordinator(BaseCoordinator): 'retry_backoff_ms': 100, 'api_version': (0, 10, 1), 'exclude_internal_topics': True, - 'metric_group_prefix': 'consumer' + 'metric_group_prefix': 'consumer', + 'coordinator_not_ready_retry_timeout_ms': None } def __init__(self, client, subscription, metrics, **configs): @@ -77,6 +78,10 @@ def __init__(self, client, subscription, metrics, **configs): (such as offsets) should be exposed to the consumer. If set to True the only way to receive records from an internal topic is subscribing to it. Requires 0.10+. Default: True + coordinator_not_ready_retry_timeout_ms (int): The timeout used to + detect that the Kafka coordinator is not available. If 'None', + the default behavior of polling indefinitely would be kept. + Default: None. """ super(ConsumerCoordinator, self).__init__(client, metrics, **configs) @@ -128,7 +133,10 @@ def __init__(self, client, subscription, metrics, **configs): def __del__(self): if hasattr(self, '_cluster') and self._cluster: - self._cluster.remove_listener(WeakMethod(self._handle_metadata_update)) + try: + self._cluster.remove_listener(WeakMethod(self._handle_metadata_update)) + except TypeError: + pass super(ConsumerCoordinator, self).__del__() def protocol_type(self): @@ -572,7 +580,7 @@ def _send_offset_commit_request(self, offsets): offset_data[tp.topic][tp.partition] = offset if self._subscription.partitions_auto_assigned(): - generation = self.generation() + generation = self.generation() or Generation.NO_GENERATION else: generation = Generation.NO_GENERATION @@ -582,12 +590,40 @@ def _send_offset_commit_request(self, offsets): if self.config['api_version'] >= (0, 9) and generation is None: return Future().failure(Errors.CommitFailedError()) - if self.config['api_version'] >= (0, 9): - request = OffsetCommitRequest[2]( + version = self._client.api_version(OffsetCommitRequest, max_version=6) + if version == 0: + request = OffsetCommitRequest[version]( + self.group_id, + [( + topic, [( + partition, + offset.offset, + offset.metadata + ) for partition, offset in six.iteritems(partitions)] + ) for topic, partitions in six.iteritems(offset_data)] + ) + elif version == 1: + request = OffsetCommitRequest[version]( + self.group_id, + # This api version was only used in v0.8.2, prior to join group apis + # so this always ends up as NO_GENERATION + generation.generation_id, + generation.member_id, + [( + topic, [( + partition, + offset.offset, + -1, # timestamp, unused + offset.metadata + ) for partition, offset in six.iteritems(partitions)] + ) for topic, partitions in six.iteritems(offset_data)] + ) + elif version <= 4: + request = OffsetCommitRequest[version]( self.group_id, generation.generation_id, generation.member_id, - OffsetCommitRequest[2].DEFAULT_RETENTION_TIME, + OffsetCommitRequest[version].DEFAULT_RETENTION_TIME, [( topic, [( partition, @@ -596,25 +632,29 @@ def _send_offset_commit_request(self, offsets): ) for partition, offset in six.iteritems(partitions)] ) for topic, partitions in six.iteritems(offset_data)] ) - elif self.config['api_version'] >= (0, 8, 2): - request = OffsetCommitRequest[1]( - self.group_id, -1, '', + elif version <= 5: + request = OffsetCommitRequest[version]( + self.group_id, + generation.generation_id, + generation.member_id, [( topic, [( partition, offset.offset, - -1, offset.metadata ) for partition, offset in six.iteritems(partitions)] ) for topic, partitions in six.iteritems(offset_data)] ) - elif self.config['api_version'] >= (0, 8, 1): - request = OffsetCommitRequest[0]( + else: + request = OffsetCommitRequest[version]( self.group_id, + generation.generation_id, + generation.member_id, [( topic, [( partition, offset.offset, + offset.leader_epoch, offset.metadata ) for partition, offset in six.iteritems(partitions)] ) for topic, partitions in six.iteritems(offset_data)] @@ -731,16 +771,13 @@ def _send_offset_fetch_request(self, partitions): for tp in partitions: topic_partitions[tp.topic].add(tp.partition) - if self.config['api_version'] >= (0, 8, 2): - request = OffsetFetchRequest[1]( - self.group_id, - list(topic_partitions.items()) - ) - else: - request = OffsetFetchRequest[0]( - self.group_id, - list(topic_partitions.items()) - ) + version = self._client.api_version(OffsetFetchRequest, max_version=5) + # Starting in version 2, the request can contain a null topics array to indicate that offsets should be fetched + # TODO: support + request = OffsetFetchRequest[version]( + self.group_id, + list(topic_partitions.items()) + ) # send the request with a callback future = Future() @@ -750,9 +787,33 @@ def _send_offset_fetch_request(self, partitions): return future def _handle_offset_fetch_response(self, future, response): + if response.API_VERSION >= 2 and response.error_code != Errors.NoError.errno: + error_type = Errors.for_code(response.error_code) + log.debug("Offset fetch failed: %s", error_type.__name__) + error = error_type() + if error_type is Errors.GroupLoadInProgressError: + # Retry + future.failure(error) + elif error_type is Errors.NotCoordinatorForGroupError: + # re-discover the coordinator and retry + self.coordinator_dead(error) + future.failure(error) + elif error_type is Errors.GroupAuthorizationFailedError: + future.failure(error) + else: + log.error("Unknown error fetching offsets: %s", error) + future.failure(error) + return + offsets = {} for topic, partitions in response.topics: - for partition, offset, metadata, error_code in partitions: + for partition_data in partitions: + partition, offset = partition_data[:2] + if response.API_VERSION >= 5: + leader_epoch, metadata, error_code = partition_data[2:] + else: + metadata, error_code = partition_data[2:] + leader_epoch = -1 tp = TopicPartition(topic, partition) error_type = Errors.for_code(error_code) if error_type is not Errors.NoError: @@ -764,7 +825,7 @@ def _handle_offset_fetch_response(self, future, response): future.failure(error) elif error_type is Errors.NotCoordinatorForGroupError: # re-discover the coordinator and retry - self.coordinator_dead(error_type()) + self.coordinator_dead(error) future.failure(error) elif error_type is Errors.UnknownTopicOrPartitionError: log.warning("OffsetFetchRequest -- unknown topic %s" @@ -779,7 +840,8 @@ def _handle_offset_fetch_response(self, future, response): elif offset >= 0: # record the position with the offset # (-1 indicates no committed offset to fetch) - offsets[tp] = OffsetAndMetadata(offset, metadata) + # TODO: save leader_epoch + offsets[tp] = OffsetAndMetadata(offset, metadata, -1) else: log.debug("Group %s has no committed offset for partition" " %s", self.group_id, tp) diff --git a/kafka/errors.py b/kafka/errors.py index b33cf51e2..aaba89d39 100644 --- a/kafka/errors.py +++ b/kafka/errors.py @@ -186,7 +186,8 @@ class ReplicaNotAvailableError(BrokerResponseError): message = 'REPLICA_NOT_AVAILABLE' description = ('If replica is expected on a broker, but is not (this can be' ' safely ignored).') - + retriable = True + invalid_metadata = True class MessageSizeTooLargeError(BrokerResponseError): errno = 10 @@ -210,10 +211,11 @@ class OffsetMetadataTooLargeError(BrokerResponseError): ' offset metadata.') -# TODO is this deprecated? https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes -class StaleLeaderEpochCodeError(BrokerResponseError): +class NetworkExceptionError(BrokerResponseError): errno = 13 - message = 'STALE_LEADER_EPOCH_CODE' + message = 'NETWORK_EXCEPTION' + retriable = True + invalid_metadata = True class GroupLoadInProgressError(BrokerResponseError): @@ -441,24 +443,597 @@ class PolicyViolationError(BrokerResponseError): errno = 44 message = 'POLICY_VIOLATION' description = 'Request parameters do not satisfy the configured policy.' + retriable = False + + +class OutOfOrderSequenceNumberError(BrokerResponseError): + errno = 45 + message = 'OUT_OF_ORDER_SEQUENCE_NUMBER' + description = 'The broker received an out of order sequence number.' + retriable = False + + +class DuplicateSequenceNumberError(BrokerResponseError): + errno = 46 + message = 'DUPLICATE_SEQUENCE_NUMBER' + description = 'The broker received a duplicate sequence number.' + retriable = False + + +class InvalidProducerEpochError(BrokerResponseError): + errno = 47 + message = 'INVALID_PRODUCER_EPOCH' + description = 'Producer attempted to produce with an old epoch.' + retriable = False + + +class InvalidTxnStateError(BrokerResponseError): + errno = 48 + message = 'INVALID_TXN_STATE' + description = 'The producer attempted a transactional operation in an invalid state.' + retriable = False + + +class InvalidProducerIdMappingError(BrokerResponseError): + errno = 49 + message = 'INVALID_PRODUCER_ID_MAPPING' + description = 'The producer attempted to use a producer id which is not currently assigned to its transactional id.' + retriable = False + + +class InvalidTransactionTimeoutError(BrokerResponseError): + errno = 50 + message = 'INVALID_TRANSACTION_TIMEOUT' + description = 'The transaction timeout is larger than the maximum value allowed by the broker (as configured by transaction.max.timeout.ms).' + retriable = False + + +class ConcurrentTransactionsError(BrokerResponseError): + errno = 51 + message = 'CONCURRENT_TRANSACTIONS' + description = 'The producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing.' + retriable = True + + +class TransactionCoordinatorFencedError(BrokerResponseError): + errno = 52 + message = 'TRANSACTION_COORDINATOR_FENCED' + description = 'Indicates that the transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer.' + retriable = False + + +class TransactionalIdAuthorizationFailedError(BrokerResponseError): + errno = 53 + message = 'TRANSACTIONAL_ID_AUTHORIZATION_FAILED' + description = 'Transactional Id authorization failed.' + retriable = False class SecurityDisabledError(BrokerResponseError): errno = 54 message = 'SECURITY_DISABLED' description = 'Security features are disabled.' + retriable = False + + +class OperationNotAttemptedError(BrokerResponseError): + errno = 55 + message = 'OPERATION_NOT_ATTEMPTED' + description = 'The broker did not attempt to execute this operation. This may happen for batched RPCs where some operations in the batch failed, causing the broker to respond without trying the rest.' + retriable = False + + +class KafkaStorageError(BrokerResponseError): + errno = 56 + message = 'KAFKA_STORAGE_ERROR' + description = 'Disk error when trying to access log file on the disk.' + retriable = True + invalid_metadata = True + + +class LogDirNotFoundError(BrokerResponseError): + errno = 57 + message = 'LOG_DIR_NOT_FOUND' + description = 'The user-specified log directory is not found in the broker config.' + retriable = False + + +class SaslAuthenticationFailedError(BrokerResponseError): + errno = 58 + message = 'SASL_AUTHENTICATION_FAILED' + description = 'SASL Authentication failed.' + retriable = False + + +class UnknownProducerIdError(BrokerResponseError): + errno = 59 + message = 'UNKNOWN_PRODUCER_ID' + description = 'This exception is raised by the broker if it could not locate the producer metadata associated with the producerId in question. This could happen if, for instance, the producer\'s records were deleted because their retention time had elapsed. Once the last records of the producerId are removed, the producer\'s metadata is removed from the broker, and future appends by the producer will return this exception.' + retriable = False + + +class ReassignmentInProgressError(BrokerResponseError): + errno = 60 + message = 'REASSIGNMENT_IN_PROGRESS' + description = 'A partition reassignment is in progress.' + retriable = False + + +class DelegationTokenAuthDisabledError(BrokerResponseError): + errno = 61 + message = 'DELEGATION_TOKEN_AUTH_DISABLED' + description = 'Delegation Token feature is not enabled.' + retriable = False + + +class DelegationTokenNotFoundError(BrokerResponseError): + errno = 62 + message = 'DELEGATION_TOKEN_NOT_FOUND' + description = 'Delegation Token is not found on server.' + retriable = False + + +class DelegationTokenOwnerMismatchError(BrokerResponseError): + errno = 63 + message = 'DELEGATION_TOKEN_OWNER_MISMATCH' + description = 'Specified Principal is not valid Owner/Renewer.' + retriable = False + + +class DelegationTokenRequestNotAllowedError(BrokerResponseError): + errno = 64 + message = 'DELEGATION_TOKEN_REQUEST_NOT_ALLOWED' + description = 'Delegation Token requests are not allowed on PLAINTEXT/1-way SSL channels and on delegation token authenticated channels.' + retriable = False + + +class DelegationTokenAuthorizationFailedError(BrokerResponseError): + errno = 65 + message = 'DELEGATION_TOKEN_AUTHORIZATION_FAILED' + description = 'Delegation Token authorization failed.' + retriable = False + + +class DelegationTokenExpiredError(BrokerResponseError): + errno = 66 + message = 'DELEGATION_TOKEN_EXPIRED' + description = 'Delegation Token is expired.' + retriable = False + + +class InvalidPrincipalTypeError(BrokerResponseError): + errno = 67 + message = 'INVALID_PRINCIPAL_TYPE' + description = 'Supplied principalType is not supported.' + retriable = False class NonEmptyGroupError(BrokerResponseError): errno = 68 message = 'NON_EMPTY_GROUP' description = 'The group is not empty.' + retriable = False class GroupIdNotFoundError(BrokerResponseError): errno = 69 message = 'GROUP_ID_NOT_FOUND' description = 'The group id does not exist.' + retriable = False + + +class FetchSessionIdNotFoundError(BrokerResponseError): + errno = 70 + message = 'FETCH_SESSION_ID_NOT_FOUND' + description = 'The fetch session ID was not found.' + retriable = True + + +class InvalidFetchSessionEpochError(BrokerResponseError): + errno = 71 + message = 'INVALID_FETCH_SESSION_EPOCH' + description = 'The fetch session epoch is invalid.' + retriable = True + + +class ListenerNotFoundError(BrokerResponseError): + errno = 72 + message = 'LISTENER_NOT_FOUND' + description = 'There is no listener on the leader broker that matches the listener on which metadata request was processed.' + retriable = True + invalid_metadata = True + + +class TopicDeletionDisabledError(BrokerResponseError): + errno = 73 + message = 'TOPIC_DELETION_DISABLED' + description = 'Topic deletion is disabled.' + retriable = False + + +class FencedLeaderEpochError(BrokerResponseError): + errno = 74 + message = 'FENCED_LEADER_EPOCH' + description = 'The leader epoch in the request is older than the epoch on the broker.' + retriable = True + invalid_metadata = True + + +class UnknownLeaderEpochError(BrokerResponseError): + errno = 75 + message = 'UNKNOWN_LEADER_EPOCH' + description = 'The leader epoch in the request is newer than the epoch on the broker.' + retriable = True + invalid_metadata = True + + +class UnsupportedCompressionTypeError(BrokerResponseError): + errno = 76 + message = 'UNSUPPORTED_COMPRESSION_TYPE' + description = 'The requesting client does not support the compression type of given partition.' + retriable = False + + +class StaleBrokerEpochError(BrokerResponseError): + errno = 77 + message = 'STALE_BROKER_EPOCH' + description = 'Broker epoch has changed.' + retriable = False + + +class OffsetNotAvailableError(BrokerResponseError): + errno = 78 + message = 'OFFSET_NOT_AVAILABLE' + description = 'The leader high watermark has not caught up from a recent leader election so the offsets cannot be guaranteed to be monotonically increasing.' + retriable = True + + +class MemberIdRequiredError(BrokerResponseError): + errno = 79 + message = 'MEMBER_ID_REQUIRED' + description = 'The group member needs to have a valid member id before actually entering a consumer group.' + retriable = False + + +class PreferredLeaderNotAvailableError(BrokerResponseError): + errno = 80 + message = 'PREFERRED_LEADER_NOT_AVAILABLE' + description = 'The preferred leader was not available.' + retriable = True + invalid_metadata = True + + +class GroupMaxSizeReachedError(BrokerResponseError): + errno = 81 + message = 'GROUP_MAX_SIZE_REACHED' + description = 'The consumer group has reached its max size.' + retriable = False + + +class FencedInstanceIdError(BrokerResponseError): + errno = 82 + message = 'FENCED_INSTANCE_ID' + description = 'The broker rejected this static consumer since another consumer with the same group.instance.id has registered with a different member.id.' + retriable = False + + +class EligibleLeadersNotAvailableError(BrokerResponseError): + errno = 83 + message = 'ELIGIBLE_LEADERS_NOT_AVAILABLE' + description = 'Eligible topic partition leaders are not available.' + retriable = True + invalid_metadata = True + + +class ElectionNotNeededError(BrokerResponseError): + errno = 84 + message = 'ELECTION_NOT_NEEDED' + description = 'Leader election not needed for topic partition.' + retriable = True + invalid_metadata = True + + +class NoReassignmentInProgressError(BrokerResponseError): + errno = 85 + message = 'NO_REASSIGNMENT_IN_PROGRESS' + description = 'No partition reassignment is in progress.' + retriable = False + + +class GroupSubscribedToTopicError(BrokerResponseError): + errno = 86 + message = 'GROUP_SUBSCRIBED_TO_TOPIC' + description = 'Deleting offsets of a topic is forbidden while the consumer group is actively subscribed to it.' + retriable = False + + +class InvalidRecordError(BrokerResponseError): + errno = 87 + message = 'INVALID_RECORD' + description = 'This record has failed the validation on broker and hence will be rejected.' + retriable = False + + +class UnstableOffsetCommitError(BrokerResponseError): + errno = 88 + message = 'UNSTABLE_OFFSET_COMMIT' + description = 'There are unstable offsets that need to be cleared.' + retriable = True + + +class ThrottlingQuotaExceededError(BrokerResponseError): + errno = 89 + message = 'THROTTLING_QUOTA_EXCEEDED' + description = 'The throttling quota has been exceeded.' + retriable = True + + +class ProducerFencedError(BrokerResponseError): + errno = 90 + message = 'PRODUCER_FENCED' + description = 'There is a newer producer with the same transactionalId which fences the current one.' + retriable = False + + +class ResourceNotFoundError(BrokerResponseError): + errno = 91 + message = 'RESOURCE_NOT_FOUND' + description = 'A request illegally referred to a resource that does not exist.' + retriable = False + + +class DuplicateResourceError(BrokerResponseError): + errno = 92 + message = 'DUPLICATE_RESOURCE' + description = 'A request illegally referred to the same resource twice.' + retriable = False + + +class UnacceptableCredentialError(BrokerResponseError): + errno = 93 + message = 'UNACCEPTABLE_CREDENTIAL' + description = 'Requested credential would not meet criteria for acceptability.' + retriable = False + + +class InconsistentVoterSetError(BrokerResponseError): + errno = 94 + message = 'INCONSISTENT_VOTER_SET' + description = 'Indicates that the either the sender or recipient of a voter-only request is not one of the expected voters.' + retriable = False + + +class InvalidUpdateVersionError(BrokerResponseError): + errno = 95 + message = 'INVALID_UPDATE_VERSION' + description = 'The given update version was invalid.' + retriable = False + + +class FeatureUpdateFailedError(BrokerResponseError): + errno = 96 + message = 'FEATURE_UPDATE_FAILED' + description = 'Unable to update finalized features due to an unexpected server error.' + retriable = False + + +class PrincipalDeserializationFailureError(BrokerResponseError): + errno = 97 + message = 'PRINCIPAL_DESERIALIZATION_FAILURE' + description = 'Request principal deserialization failed during forwarding. This indicates an internal error on the broker cluster security setup.' + retriable = False + + +class SnapshotNotFoundError(BrokerResponseError): + errno = 98 + message = 'SNAPSHOT_NOT_FOUND' + description = 'Requested snapshot was not found.' + retriable = False + + +class PositionOutOfRangeError(BrokerResponseError): + errno = 99 + message = 'POSITION_OUT_OF_RANGE' + description = 'Requested position is not greater than or equal to zero, and less than the size of the snapshot.' + retriable = False + + +class UnknownTopicIdError(BrokerResponseError): + errno = 100 + message = 'UNKNOWN_TOPIC_ID' + description = 'This server does not host this topic ID.' + retriable = True + invalid_metadata = True + + +class DuplicateBrokerRegistrationError(BrokerResponseError): + errno = 101 + message = 'DUPLICATE_BROKER_REGISTRATION' + description = 'This broker ID is already in use.' + retriable = False + + +class BrokerIdNotRegisteredError(BrokerResponseError): + errno = 102 + message = 'BROKER_ID_NOT_REGISTERED' + description = 'The given broker ID was not registered.' + retriable = False + + +class InconsistentTopicIdError(BrokerResponseError): + errno = 103 + message = 'INCONSISTENT_TOPIC_ID' + description = 'The log\'s topic ID did not match the topic ID in the request.' + retriable = True + invalid_metadata = True + + +class InconsistentClusterIdError(BrokerResponseError): + errno = 104 + message = 'INCONSISTENT_CLUSTER_ID' + description = 'The clusterId in the request does not match that found on the server.' + retriable = False + + +class TransactionalIdNotFoundError(BrokerResponseError): + errno = 105 + message = 'TRANSACTIONAL_ID_NOT_FOUND' + description = 'The transactionalId could not be found.' + retriable = False + + +class FetchSessionTopicIdError(BrokerResponseError): + errno = 106 + message = 'FETCH_SESSION_TOPIC_ID_ERROR' + description = 'The fetch session encountered inconsistent topic ID usage.' + retriable = True + + +class IneligibleReplicaError(BrokerResponseError): + errno = 107 + message = 'INELIGIBLE_REPLICA' + description = 'The new ISR contains at least one ineligible replica.' + retriable = False + + +class NewLeaderElectedError(BrokerResponseError): + errno = 108 + message = 'NEW_LEADER_ELECTED' + description = 'The AlterPartition request successfully updated the partition state but the leader has changed.' + retriable = False + + +class OffsetMovedToTieredStorageError(BrokerResponseError): + errno = 109 + message = 'OFFSET_MOVED_TO_TIERED_STORAGE' + description = 'The requested offset is moved to tiered storage.' + retriable = False + + +class FencedMemberEpochError(BrokerResponseError): + errno = 110 + message = 'FENCED_MEMBER_EPOCH' + description = 'The member epoch is fenced by the group coordinator. The member must abandon all its partitions and rejoin.' + retriable = False + + +class UnreleasedInstanceIdError(BrokerResponseError): + errno = 111 + message = 'UNRELEASED_INSTANCE_ID' + description = 'The instance ID is still used by another member in the consumer group. That member must leave first.' + retriable = False + + +class UnsupportedAssignorError(BrokerResponseError): + errno = 112 + message = 'UNSUPPORTED_ASSIGNOR' + description = 'The assignor or its version range is not supported by the consumer group.' + retriable = False + + +class StaleMemberEpochError(BrokerResponseError): + errno = 113 + message = 'STALE_MEMBER_EPOCH' + description = 'The member epoch is stale. The member must retry after receiving its updated member epoch via the ConsumerGroupHeartbeat API.' + retriable = False + + +class MismatchedEndpointTypeError(BrokerResponseError): + errno = 114 + message = 'MISMATCHED_ENDPOINT_TYPE' + description = 'The request was sent to an endpoint of the wrong type.' + retriable = False + + +class UnsupportedEndpointTypeError(BrokerResponseError): + errno = 115 + message = 'UNSUPPORTED_ENDPOINT_TYPE' + description = 'This endpoint type is not supported yet.' + retriable = False + + +class UnknownControllerIdError(BrokerResponseError): + errno = 116 + message = 'UNKNOWN_CONTROLLER_ID' + description = 'This controller ID is not known.' + retriable = False + + +class UnknownSubscriptionIdError(BrokerResponseError): + errno = 117 + message = 'UNKNOWN_SUBSCRIPTION_ID' + description = 'Client sent a push telemetry request with an invalid or outdated subscription ID.' + retriable = False + + +class TelemetryTooLargeError(BrokerResponseError): + errno = 118 + message = 'TELEMETRY_TOO_LARGE' + description = 'Client sent a push telemetry request larger than the maximum size the broker will accept.' + retriable = False + + +class InvalidRegistrationError(BrokerResponseError): + errno = 119 + message = 'INVALID_REGISTRATION' + description = 'The controller has considered the broker registration to be invalid.' + retriable = False + + +class TransactionAbortableError(BrokerResponseError): + errno = 120 + message = 'TRANSACTION_ABORTABLE' + description = 'The server encountered an error with the transaction. The client can abort the transaction to continue using this transactional ID.' + retriable = False + + +class InvalidRecordStateError(BrokerResponseError): + errno = 121 + message = 'INVALID_RECORD_STATE' + description = 'The record state is invalid. The acknowledgement of delivery could not be completed.' + retriable = False + + +class ShareSessionNotFoundError(BrokerResponseError): + errno = 122 + message = 'SHARE_SESSION_NOT_FOUND' + description = 'The share session was not found.' + retriable = True + + +class InvalidShareSessionEpochError(BrokerResponseError): + errno = 123 + message = 'INVALID_SHARE_SESSION_EPOCH' + description = 'The share session epoch is invalid.' + retriable = True + + +class FencedStateEpochError(BrokerResponseError): + errno = 124 + message = 'FENCED_STATE_EPOCH' + description = 'The share coordinator rejected the request because the share-group state epoch did not match.' + retriable = False + + +class InvalidVoterKeyError(BrokerResponseError): + errno = 125 + message = 'INVALID_VOTER_KEY' + description = 'The voter key doesn\'t match the receiving replica\'s key.' + retriable = False + + +class DuplicateVoterError(BrokerResponseError): + errno = 126 + message = 'DUPLICATE_VOTER' + description = 'The voter is already part of the set of voters.' + retriable = False + + +class VoterNotFoundError(BrokerResponseError): + errno = 127 + message = 'VOTER_NOT_FOUND' + description = 'The voter is not part of the set of voters.' + retriable = False class KafkaUnavailableError(KafkaError): @@ -512,27 +1087,12 @@ def _iter_broker_errors(): def for_code(error_code): - return kafka_errors.get(error_code, UnknownError) - - -def check_error(response): - if isinstance(response, Exception): - raise response - if response.error: - error_class = kafka_errors.get(response.error, UnknownError) - raise error_class(response) - - -RETRY_BACKOFF_ERROR_TYPES = ( - KafkaUnavailableError, LeaderNotAvailableError, - KafkaConnectionError, FailedPayloadsError -) - - -RETRY_REFRESH_ERROR_TYPES = ( - NotLeaderForPartitionError, UnknownTopicOrPartitionError, - LeaderNotAvailableError, KafkaConnectionError -) - - -RETRY_ERROR_TYPES = RETRY_BACKOFF_ERROR_TYPES + RETRY_REFRESH_ERROR_TYPES + if error_code in kafka_errors: + return kafka_errors[error_code] + else: + # The broker error code was not found in our list. This can happen when connecting + # to a newer broker (with new error codes), or simply because our error list is + # not complete. + # + # To avoid dropping the error code, create a dynamic error class w/ errno override. + return type('UnrecognizedBrokerError', (UnknownError,), {'errno': error_code}) diff --git a/kafka/metrics/metric_name.py b/kafka/metrics/metric_name.py index b5acd1662..32a7e3a4b 100644 --- a/kafka/metrics/metric_name.py +++ b/kafka/metrics/metric_name.py @@ -93,7 +93,7 @@ def __eq__(self, other): return True if other is None: return False - return (type(self) == type(other) and + return (isinstance(self, type(other)) and self.group == other.group and self.name == other.name and self.tags == other.tags) diff --git a/kafka/metrics/quota.py b/kafka/metrics/quota.py index 4d1b0d6cb..237edf841 100644 --- a/kafka/metrics/quota.py +++ b/kafka/metrics/quota.py @@ -34,7 +34,7 @@ def __hash__(self): def __eq__(self, other): if self is other: return True - return (type(self) == type(other) and + return (isinstance(self, type(other)) and self.bound == other.bound and self.is_upper_bound() == other.is_upper_bound()) diff --git a/kafka/producer/kafka.py b/kafka/producer/kafka.py index dd1cc508c..2a70700c4 100644 --- a/kafka/producer/kafka.py +++ b/kafka/producer/kafka.py @@ -188,6 +188,9 @@ class KafkaProducer(object): This setting will limit the number of record batches the producer will send in a single request to avoid sending huge requests. Default: 1048576. + allow_auto_create_topics (bool): Enable/disable auto topic creation + on metadata request. Only available with api_version >= (0, 11). + Default: True metadata_max_age_ms (int): The period of time in milliseconds after which we force a refresh of metadata even if we haven't seen any partition leadership changes to proactively discover any new @@ -216,7 +219,7 @@ class KafkaProducer(object): reconnection attempts will continue periodically with this fixed rate. To avoid connection storms, a randomization factor of 0.2 will be applied to the backoff resulting in a random range between - 20% below and 20% above the computed value. Default: 1000. + 20% below and 20% above the computed value. Default: 30000. max_in_flight_requests_per_connection (int): Requests are pipelined to kafka brokers up to this number of maximum requests per broker connection. Note that if this setting is set to be greater @@ -252,11 +255,24 @@ class KafkaProducer(object): or other configuration forbids use of all the specified ciphers), an ssl.SSLError will be raised. See ssl.SSLContext.set_ciphers api_version (tuple): Specify which Kafka API version to use. If set to - None, the client will attempt to infer the broker version by probing - various APIs. Example: (0, 10, 2). Default: None + None, the client will attempt to determine the broker version via + ApiVersionsRequest API or, for brokers earlier than 0.10, probing + various known APIs. Dynamic version checking is performed eagerly + during __init__ and can raise NoBrokersAvailableError if no connection + was made before timeout (see api_version_auto_timeout_ms below). + Different versions enable different functionality. + + Examples: + (3, 9) most recent broker release, enable all supported features + (0, 11) enables message format v2 (internal) + (0, 10, 0) enables sasl authentication and message format v1 + (0, 8, 0) enables basic functionality only + + Default: None api_version_auto_timeout_ms (int): number of milliseconds to throw a timeout exception from the constructor when checking the broker api version. Only applies if api_version set to None. + Default: 2000 metric_reporters (list): A list of classes to use as metrics reporters. Implementing the AbstractMetricsReporter interface allows plugging in classes that will be notified of new metric creation. Default: [] @@ -274,6 +290,9 @@ class KafkaProducer(object): Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms. sasl_plain_password (str): password for sasl PLAIN and SCRAM authentication. Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms. + sasl_kerberos_name (str or gssapi.Name): Constructed gssapi.Name for use with + sasl mechanism handshake. If provided, sasl_kerberos_service_name and + sasl_kerberos_domain name are ignored. Default: None. sasl_kerberos_service_name (str): Service name to include in GSSAPI sasl mechanism handshake. Default: 'kafka' sasl_kerberos_domain_name (str): kerberos domain name to use in GSSAPI @@ -302,6 +321,7 @@ class KafkaProducer(object): 'connections_max_idle_ms': 9 * 60 * 1000, 'max_block_ms': 60000, 'max_request_size': 1048576, + 'allow_auto_create_topics': True, 'metadata_max_age_ms': 300000, 'retry_backoff_ms': 100, 'request_timeout_ms': 30000, @@ -311,7 +331,7 @@ class KafkaProducer(object): 'sock_chunk_bytes': 4096, # undocumented experimental option 'sock_chunk_buffer_count': 1000, # undocumented experimental option 'reconnect_backoff_ms': 50, - 'reconnect_backoff_max_ms': 1000, + 'reconnect_backoff_max_ms': 30000, 'max_in_flight_requests_per_connection': 5, 'security_protocol': 'PLAINTEXT', 'ssl_context': None, @@ -331,6 +351,7 @@ class KafkaProducer(object): 'sasl_mechanism': None, 'sasl_plain_username': None, 'sasl_plain_password': None, + 'sasl_kerberos_name': None, 'sasl_kerberos_service_name': 'kafka', 'sasl_kerberos_domain_name': None, 'sasl_oauth_token_provider': None, @@ -385,15 +406,14 @@ def __init__(self, **configs): wakeup_timeout_ms=self.config['max_block_ms'], **self.config) - # Get auto-discovered version from client if necessary - if self.config['api_version'] is None: - self.config['api_version'] = client.config['api_version'] + # Get auto-discovered / normalized version from client + self.config['api_version'] = client.config['api_version'] if self.config['compression_type'] == 'lz4': assert self.config['api_version'] >= (0, 8, 2), 'LZ4 Requires >= Kafka 0.8.2 Brokers' if self.config['compression_type'] == 'zstd': - assert self.config['api_version'] >= (2, 1, 0), 'Zstd Requires >= Kafka 2.1.0 Brokers' + assert self.config['api_version'] >= (2, 1), 'Zstd Requires >= Kafka 2.1 Brokers' # Check compression_type for library support ct = self.config['compression_type'] @@ -524,7 +544,7 @@ def partitions_for(self, topic): def _max_usable_produce_magic(self): if self.config['api_version'] >= (0, 11): return 2 - elif self.config['api_version'] >= (0, 10): + elif self.config['api_version'] >= (0, 10, 0): return 1 else: return 0 @@ -592,8 +612,8 @@ def send(self, topic, value=None, key=None, headers=None, partition=None, timest if headers is None: headers = [] - assert type(headers) == list - assert all(type(item) == tuple and len(item) == 2 and type(item[0]) == str and type(item[1]) == bytes for item in headers) + assert isinstance(headers, list) + assert all(isinstance(item, tuple) and len(item) == 2 and isinstance(item[0], str) and isinstance(item[1], bytes) for item in headers) message_size = self._estimate_size_in_bytes(key_bytes, value_bytes, headers) self._ensure_valid_record_size(message_size) @@ -615,7 +635,7 @@ def send(self, topic, value=None, key=None, headers=None, partition=None, timest # for API exceptions return them in the future, # for other exceptions raise directly except Errors.BrokerResponseError as e: - log.debug("Exception occurred during message send: %s", e) + log.error("Exception occurred during message send: %s", e) return FutureRecordMetadata( FutureProduceResult(TopicPartition(topic, partition)), -1, None, None, diff --git a/kafka/producer/record_accumulator.py b/kafka/producer/record_accumulator.py index a2aa0e8ec..f13c21b9f 100644 --- a/kafka/producer/record_accumulator.py +++ b/kafka/producer/record_accumulator.py @@ -156,7 +156,7 @@ class RecordAccumulator(object): will also impact the compression ratio (more batching means better compression). Default: None. linger_ms (int): An artificial delay time to add before declaring a - messageset (that isn't full) ready for sending. This allows + record batch (that isn't full) ready for sending. This allows time for more records to arrive. Setting a non-zero linger_ms will trade off some latency for potentially better throughput due to more batching (and hence fewer, larger requests). diff --git a/kafka/producer/sender.py b/kafka/producer/sender.py index 35688d3f1..3dd52ba76 100644 --- a/kafka/producer/sender.py +++ b/kafka/producer/sender.py @@ -31,7 +31,6 @@ class Sender(threading.Thread): 'request_timeout_ms': 30000, 'guarantee_message_order': False, 'client_id': 'kafka-python-' + __version__, - 'api_version': (0, 8, 0), } def __init__(self, client, metadata, accumulator, metrics, **configs): @@ -103,14 +102,14 @@ def run_once(self): self._metadata.request_update() # remove any nodes we aren't ready to send to - not_ready_timeout = float('inf') + not_ready_timeout_ms = float('inf') for node in list(ready_nodes): if not self._client.is_ready(node): - log.debug('Node %s not ready; delaying produce of accumulated batch', node) + node_delay_ms = self._client.connection_delay(node) + log.debug('Node %s not ready; delaying produce of accumulated batch (%f ms)', node, node_delay_ms) self._client.maybe_connect(node, wakeup=False) ready_nodes.remove(node) - not_ready_timeout = min(not_ready_timeout, - self._client.connection_delay(node)) + not_ready_timeout_ms = min(not_ready_timeout_ms, node_delay_ms) # create produce requests batches_by_node = self._accumulator.drain( @@ -136,7 +135,7 @@ def run_once(self): # off). Note that this specifically does not include nodes with # sendable data that aren't ready to send since they would cause busy # looping. - poll_timeout_ms = min(next_ready_check_delay * 1000, not_ready_timeout) + poll_timeout_ms = min(next_ready_check_delay * 1000, not_ready_timeout_ms) if ready_nodes: log.debug("Nodes with data ready to send: %s", ready_nodes) # trace log.debug("Created %d produce requests: %s", len(requests), requests) # trace @@ -181,7 +180,7 @@ def add_topic(self, topic): self.wakeup() def _failed_produce(self, batches, node_id, error): - log.debug("Error sending produce request to node %d: %s", node_id, error) # trace + log.error("Error sending produce request to node %d: %s", node_id, error) # trace for batch in batches: self._complete_batch(batch, error, -1, None) @@ -212,9 +211,6 @@ def _handle_produce_response(self, node_id, send_time, batches, response): batch = batches_by_partition[tp] self._complete_batch(batch, error, offset, ts, log_start_offset, global_error) - if response.API_VERSION > 0: - self._sensors.record_throttle_time(response.throttle_time_ms, node=node_id) - else: # this is the acks = 0 case, just complete all requests for batch in batches: @@ -278,7 +274,7 @@ def _create_produce_requests(self, collated): collated: {node_id: [RecordBatch]} Returns: - dict: {node_id: ProduceRequest} (version depends on api_version) + dict: {node_id: ProduceRequest} (version depends on client api_versions) """ requests = {} for node_id, batches in six.iteritems(collated): @@ -291,7 +287,7 @@ def _produce_request(self, node_id, acks, timeout, batches): """Create a produce request from the given record batches. Returns: - ProduceRequest (version depends on api_version) + ProduceRequest (version depends on client api_versions) """ produce_records_by_partition = collections.defaultdict(dict) for batch in batches: @@ -301,31 +297,14 @@ def _produce_request(self, node_id, acks, timeout, batches): buf = batch.records.buffer() produce_records_by_partition[topic][partition] = buf - kwargs = {} - if self.config['api_version'] >= (2, 1): - version = 7 - elif self.config['api_version'] >= (2, 0): - version = 6 - elif self.config['api_version'] >= (1, 1): - version = 5 - elif self.config['api_version'] >= (1, 0): - version = 4 - elif self.config['api_version'] >= (0, 11): - version = 3 - kwargs = dict(transactional_id=None) - elif self.config['api_version'] >= (0, 10): - version = 2 - elif self.config['api_version'] == (0, 9): - version = 1 - else: - version = 0 + version = self._client.api_version(ProduceRequest, max_version=7) + # TODO: support transactional_id return ProduceRequest[version]( required_acks=acks, timeout=timeout, topics=[(topic, list(partition_info.items())) for topic, partition_info in six.iteritems(produce_records_by_partition)], - **kwargs ) def wakeup(self): @@ -367,15 +346,6 @@ def __init__(self, metrics, client, metadata): sensor_name=sensor_name, description='The maximum time in ms record batches spent in the record accumulator.') - sensor_name = 'produce-throttle-time' - self.produce_throttle_time_sensor = self.metrics.sensor(sensor_name) - self.add_metric('produce-throttle-time-avg', Avg(), - sensor_name=sensor_name, - description='The average throttle time in ms') - self.add_metric('produce-throttle-time-max', Max(), - sensor_name=sensor_name, - description='The maximum throttle time in ms') - sensor_name = 'records-per-request' self.records_per_request_sensor = self.metrics.sensor(sensor_name) self.add_metric('record-send-rate', Rate(), @@ -512,6 +482,3 @@ def record_errors(self, topic, count): sensor = self.metrics.get_sensor('topic.' + topic + '.record-errors') if sensor: sensor.record(count) - - def record_throttle_time(self, throttle_time_ms, node=None): - self.produce_throttle_time_sensor.record(throttle_time_ms) diff --git a/kafka/protocol/admin.py b/kafka/protocol/admin.py index f9d61e5cd..058325cb1 100644 --- a/kafka/protocol/admin.py +++ b/kafka/protocol/admin.py @@ -4,66 +4,6 @@ from kafka.protocol.types import Array, Boolean, Bytes, Int8, Int16, Int32, Int64, Schema, String, Float64, CompactString, CompactArray, TaggedFields -class ApiVersionResponse_v0(Response): - API_KEY = 18 - API_VERSION = 0 - SCHEMA = Schema( - ('error_code', Int16), - ('api_versions', Array( - ('api_key', Int16), - ('min_version', Int16), - ('max_version', Int16))) - ) - - -class ApiVersionResponse_v1(Response): - API_KEY = 18 - API_VERSION = 1 - SCHEMA = Schema( - ('error_code', Int16), - ('api_versions', Array( - ('api_key', Int16), - ('min_version', Int16), - ('max_version', Int16))), - ('throttle_time_ms', Int32) - ) - - -class ApiVersionResponse_v2(Response): - API_KEY = 18 - API_VERSION = 2 - SCHEMA = ApiVersionResponse_v1.SCHEMA - - -class ApiVersionRequest_v0(Request): - API_KEY = 18 - API_VERSION = 0 - RESPONSE_TYPE = ApiVersionResponse_v0 - SCHEMA = Schema() - - -class ApiVersionRequest_v1(Request): - API_KEY = 18 - API_VERSION = 1 - RESPONSE_TYPE = ApiVersionResponse_v1 - SCHEMA = ApiVersionRequest_v0.SCHEMA - - -class ApiVersionRequest_v2(Request): - API_KEY = 18 - API_VERSION = 2 - RESPONSE_TYPE = ApiVersionResponse_v1 - SCHEMA = ApiVersionRequest_v0.SCHEMA - - -ApiVersionRequest = [ - ApiVersionRequest_v0, ApiVersionRequest_v1, ApiVersionRequest_v2, -] -ApiVersionResponse = [ - ApiVersionResponse_v0, ApiVersionResponse_v1, ApiVersionResponse_v2, -] - - class CreateTopicsResponse_v0(Response): API_KEY = 19 API_VERSION = 0 @@ -406,41 +346,6 @@ class DescribeGroupsRequest_v3(Request): ] -class SaslHandShakeResponse_v0(Response): - API_KEY = 17 - API_VERSION = 0 - SCHEMA = Schema( - ('error_code', Int16), - ('enabled_mechanisms', Array(String('utf-8'))) - ) - - -class SaslHandShakeResponse_v1(Response): - API_KEY = 17 - API_VERSION = 1 - SCHEMA = SaslHandShakeResponse_v0.SCHEMA - - -class SaslHandShakeRequest_v0(Request): - API_KEY = 17 - API_VERSION = 0 - RESPONSE_TYPE = SaslHandShakeResponse_v0 - SCHEMA = Schema( - ('mechanism', String('utf-8')) - ) - - -class SaslHandShakeRequest_v1(Request): - API_KEY = 17 - API_VERSION = 1 - RESPONSE_TYPE = SaslHandShakeResponse_v1 - SCHEMA = SaslHandShakeRequest_v0.SCHEMA - - -SaslHandShakeRequest = [SaslHandShakeRequest_v0, SaslHandShakeRequest_v1] -SaslHandShakeResponse = [SaslHandShakeResponse_v0, SaslHandShakeResponse_v1] - - class DescribeAclsResponse_v0(Response): API_KEY = 29 API_VERSION = 0 @@ -523,8 +428,8 @@ class DescribeAclsRequest_v2(Request): SCHEMA = DescribeAclsRequest_v1.SCHEMA -DescribeAclsRequest = [DescribeAclsRequest_v0, DescribeAclsRequest_v1] -DescribeAclsResponse = [DescribeAclsResponse_v0, DescribeAclsResponse_v1] +DescribeAclsRequest = [DescribeAclsRequest_v0, DescribeAclsRequest_v1, DescribeAclsRequest_v2] +DescribeAclsResponse = [DescribeAclsResponse_v0, DescribeAclsResponse_v1, DescribeAclsResponse_v2] class CreateAclsResponse_v0(Response): API_KEY = 30 @@ -719,7 +624,7 @@ class DescribeConfigsResponse_v1(Response): ('config_names', String('utf-8')), ('config_value', String('utf-8')), ('read_only', Boolean), - ('is_default', Boolean), + ('config_source', Int8), ('is_sensitive', Boolean), ('config_synonyms', Array( ('config_name', String('utf-8')), @@ -790,6 +695,48 @@ class DescribeConfigsRequest_v2(Request): ] +class DescribeLogDirsResponse_v0(Response): + API_KEY = 35 + API_VERSION = 0 + FLEXIBLE_VERSION = True + SCHEMA = Schema( + ('throttle_time_ms', Int32), + ('log_dirs', Array( + ('error_code', Int16), + ('log_dir', String('utf-8')), + ('topics', Array( + ('name', String('utf-8')), + ('partitions', Array( + ('partition_index', Int32), + ('partition_size', Int64), + ('offset_lag', Int64), + ('is_future_key', Boolean) + )) + )) + )) + ) + + +class DescribeLogDirsRequest_v0(Request): + API_KEY = 35 + API_VERSION = 0 + RESPONSE_TYPE = DescribeLogDirsResponse_v0 + SCHEMA = Schema( + ('topics', Array( + ('topic', String('utf-8')), + ('partitions', Int32) + )) + ) + + +DescribeLogDirsResponse = [ + DescribeLogDirsResponse_v0, +] +DescribeLogDirsRequest = [ + DescribeLogDirsRequest_v0, +] + + class SaslAuthenticateResponse_v0(Response): API_KEY = 36 API_VERSION = 0 @@ -925,7 +872,7 @@ class DeleteGroupsRequest_v1(Request): ] -class DescribeClientQuotasResponse_v0(Request): +class DescribeClientQuotasResponse_v0(Response): API_KEY = 48 API_VERSION = 0 SCHEMA = Schema( diff --git a/kafka/protocol/api_versions.py b/kafka/protocol/api_versions.py new file mode 100644 index 000000000..7e2e61251 --- /dev/null +++ b/kafka/protocol/api_versions.py @@ -0,0 +1,90 @@ +from __future__ import absolute_import + +from io import BytesIO + +from kafka.protocol.api import Request, Response +from kafka.protocol.types import Array, Int16, Int32, Schema + + +class BaseApiVersionsResponse(Response): + API_KEY = 18 + API_VERSION = 0 + SCHEMA = Schema( + ('error_code', Int16), + ('api_versions', Array( + ('api_key', Int16), + ('min_version', Int16), + ('max_version', Int16))) + ) + + @classmethod + def decode(cls, data): + if isinstance(data, bytes): + data = BytesIO(data) + # Check error_code, decode as v0 if any error + curr = data.tell() + err = Int16.decode(data) + data.seek(curr) + if err != 0: + return ApiVersionsResponse_v0.decode(data) + return super(BaseApiVersionsResponse, cls).decode(data) + + +class ApiVersionsResponse_v0(Response): + API_KEY = 18 + API_VERSION = 0 + SCHEMA = Schema( + ('error_code', Int16), + ('api_versions', Array( + ('api_key', Int16), + ('min_version', Int16), + ('max_version', Int16))) + ) + + +class ApiVersionsResponse_v1(BaseApiVersionsResponse): + API_KEY = 18 + API_VERSION = 1 + SCHEMA = Schema( + ('error_code', Int16), + ('api_versions', Array( + ('api_key', Int16), + ('min_version', Int16), + ('max_version', Int16))), + ('throttle_time_ms', Int32) + ) + + +class ApiVersionsResponse_v2(BaseApiVersionsResponse): + API_KEY = 18 + API_VERSION = 2 + SCHEMA = ApiVersionsResponse_v1.SCHEMA + + +class ApiVersionsRequest_v0(Request): + API_KEY = 18 + API_VERSION = 0 + RESPONSE_TYPE = ApiVersionsResponse_v0 + SCHEMA = Schema() + + +class ApiVersionsRequest_v1(Request): + API_KEY = 18 + API_VERSION = 1 + RESPONSE_TYPE = ApiVersionsResponse_v1 + SCHEMA = ApiVersionsRequest_v0.SCHEMA + + +class ApiVersionsRequest_v2(Request): + API_KEY = 18 + API_VERSION = 2 + RESPONSE_TYPE = ApiVersionsResponse_v2 + SCHEMA = ApiVersionsRequest_v1.SCHEMA + + +ApiVersionsRequest = [ + ApiVersionsRequest_v0, ApiVersionsRequest_v1, ApiVersionsRequest_v2, +] +ApiVersionsResponse = [ + ApiVersionsResponse_v0, ApiVersionsResponse_v1, ApiVersionsResponse_v2, +] diff --git a/kafka/protocol/broker_api_versions.py b/kafka/protocol/broker_api_versions.py new file mode 100644 index 000000000..299ab547a --- /dev/null +++ b/kafka/protocol/broker_api_versions.py @@ -0,0 +1,66 @@ +BROKER_API_VERSIONS = { + # api_versions responses prior to (0, 10) are synthesized for compatibility + (0, 8, 0): {0: (0, 0), 1: (0, 0), 2: (0, 0), 3: (0, 0)}, + # adds offset commit + fetch + (0, 8, 1): {0: (0, 0), 1: (0, 0), 2: (0, 0), 3: (0, 0), 8: (0, 0), 9: (0, 0)}, + # adds find coordinator + (0, 8, 2): {0: (0, 0), 1: (0, 0), 2: (0, 0), 3: (0, 0), 8: (0, 1), 9: (0, 1), 10: (0, 0)}, + # adds group management (join/sync/leave/heartbeat) + (0, 9): {0: (0, 1), 1: (0, 1), 2: (0, 0), 3: (0, 0), 8: (0, 2), 9: (0, 1), 10: (0, 0), 11: (0, 0), 12: (0, 0), 13: (0, 0), 14: (0, 0), 15: (0, 0), 16: (0, 0)}, + # adds message format v1, sasl, and api versions api + (0, 10, 0): {0: (0, 2), 1: (0, 2), 2: (0, 0), 3: (0, 1), 4: (0, 0), 5: (0, 0), 6: (0, 2), 7: (1, 1), 8: (0, 2), 9: (0, 1), 10: (0, 0), 11: (0, 0), 12: (0, 0), 13: (0, 0), 14: (0, 0), 15: (0, 0), 16: (0, 0), 17: (0, 0), 18: (0, 0)}, + + # All data below is copied from brokers via api_versions_response (see make servers/*/api_versions) + # adds admin apis create/delete topics, and bumps fetch/listoffsets/metadata/joingroup + (0, 10, 1): {0: (0, 2), 1: (0, 3), 2: (0, 1), 3: (0, 2), 4: (0, 0), 5: (0, 0), 6: (0, 2), 7: (1, 1), 8: (0, 2), 9: (0, 1), 10: (0, 0), 11: (0, 1), 12: (0, 0), 13: (0, 0), 14: (0, 0), 15: (0, 0), 16: (0, 0), 17: (0, 0), 18: (0, 0), 19: (0, 0), 20: (0, 0)}, + + # bumps offsetfetch/create-topics + (0, 10, 2): {0: (0, 2), 1: (0, 3), 2: (0, 1), 3: (0, 2), 4: (0, 0), 5: (0, 0), 6: (0, 3), 7: (1, 1), 8: (0, 2), 9: (0, 2), 10: (0, 0), 11: (0, 1), 12: (0, 0), 13: (0, 0), 14: (0, 0), 15: (0, 0), 16: (0, 0), 17: (0, 0), 18: (0, 0), 19: (0, 1), 20: (0, 0)}, + + # Adds message format v2, and more admin apis (describe/create/delete acls, describe/alter configs, etc) + (0, 11): {0: (0, 3), 1: (0, 5), 2: (0, 2), 3: (0, 4), 4: (0, 0), 5: (0, 0), 6: (0, 3), 7: (1, 1), 8: (0, 3), 9: (0, 3), 10: (0, 1), 11: (0, 2), 12: (0, 1), 13: (0, 1), 14: (0, 1), 15: (0, 1), 16: (0, 1), 17: (0, 0), 18: (0, 1), 19: (0, 2), 20: (0, 1), 21: (0, 0), 22: (0, 0), 23: (0, 0), 24: (0, 0), 25: (0, 0), 26: (0, 0), 27: (0, 0), 28: (0, 0), 29: (0, 0), 30: (0, 0), 31: (0, 0), 32: (0, 0), 33: (0, 0)}, + + # Adds Sasl Authenticate, and additional admin apis (describe/alter log dirs, etc) + (1, 0): {0: (0, 5), 1: (0, 6), 2: (0, 2), 3: (0, 5), 4: (0, 1), 5: (0, 0), 6: (0, 4), 7: (0, 1), 8: (0, 3), 9: (0, 3), 10: (0, 1), 11: (0, 2), 12: (0, 1), 13: (0, 1), 14: (0, 1), 15: (0, 1), 16: (0, 1), 17: (0, 1), 18: (0, 1), 19: (0, 2), 20: (0, 1), 21: (0, 0), 22: (0, 0), 23: (0, 0), 24: (0, 0), 25: (0, 0), 26: (0, 0), 27: (0, 0), 28: (0, 0), 29: (0, 0), 30: (0, 0), 31: (0, 0), 32: (0, 0), 33: (0, 0), 34: (0, 0), 35: (0, 0), 36: (0, 0), 37: (0, 0)}, + + (1, 1): {0: (0, 5), 1: (0, 7), 2: (0, 2), 3: (0, 5), 4: (0, 1), 5: (0, 0), 6: (0, 4), 7: (0, 1), 8: (0, 3), 9: (0, 3), 10: (0, 1), 11: (0, 2), 12: (0, 1), 13: (0, 1), 14: (0, 1), 15: (0, 1), 16: (0, 1), 17: (0, 1), 18: (0, 1), 19: (0, 2), 20: (0, 1), 21: (0, 0), 22: (0, 0), 23: (0, 0), 24: (0, 0), 25: (0, 0), 26: (0, 0), 27: (0, 0), 28: (0, 0), 29: (0, 0), 30: (0, 0), 31: (0, 0), 32: (0, 1), 33: (0, 0), 34: (0, 0), 35: (0, 0), 36: (0, 0), 37: (0, 0), 38: (0, 0), 39: (0, 0), 40: (0, 0), 41: (0, 0), 42: (0, 0)}, + + (2, 0): {0: (0, 6), 1: (0, 8), 2: (0, 3), 3: (0, 6), 4: (0, 1), 5: (0, 0), 6: (0, 4), 7: (0, 1), 8: (0, 4), 9: (0, 4), 10: (0, 2), 11: (0, 3), 12: (0, 2), 13: (0, 2), 14: (0, 2), 15: (0, 2), 16: (0, 2), 17: (0, 1), 18: (0, 2), 19: (0, 3), 20: (0, 2), 21: (0, 1), 22: (0, 1), 23: (0, 1), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 1), 29: (0, 1), 30: (0, 1), 31: (0, 1), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 0), 37: (0, 1), 38: (0, 1), 39: (0, 1), 40: (0, 1), 41: (0, 1), 42: (0, 1)}, + + (2, 1): {0: (0, 7), 1: (0, 10), 2: (0, 4), 3: (0, 7), 4: (0, 1), 5: (0, 0), 6: (0, 4), 7: (0, 1), 8: (0, 6), 9: (0, 5), 10: (0, 2), 11: (0, 3), 12: (0, 2), 13: (0, 2), 14: (0, 2), 15: (0, 2), 16: (0, 2), 17: (0, 1), 18: (0, 2), 19: (0, 3), 20: (0, 3), 21: (0, 1), 22: (0, 1), 23: (0, 2), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 2), 29: (0, 1), 30: (0, 1), 31: (0, 1), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 0), 37: (0, 1), 38: (0, 1), 39: (0, 1), 40: (0, 1), 41: (0, 1), 42: (0, 1)}, + + (2, 2): {0: (0, 7), 1: (0, 10), 2: (0, 5), 3: (0, 7), 4: (0, 2), 5: (0, 1), 6: (0, 5), 7: (0, 2), 8: (0, 6), 9: (0, 5), 10: (0, 2), 11: (0, 4), 12: (0, 2), 13: (0, 2), 14: (0, 2), 15: (0, 2), 16: (0, 2), 17: (0, 1), 18: (0, 2), 19: (0, 3), 20: (0, 3), 21: (0, 1), 22: (0, 1), 23: (0, 2), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 2), 29: (0, 1), 30: (0, 1), 31: (0, 1), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 1), 37: (0, 1), 38: (0, 1), 39: (0, 1), 40: (0, 1), 41: (0, 1), 42: (0, 1), 43: (0, 0)}, + + (2, 3): {0: (0, 7), 1: (0, 11), 2: (0, 5), 3: (0, 8), 4: (0, 2), 5: (0, 1), 6: (0, 5), 7: (0, 2), 8: (0, 7), 9: (0, 5), 10: (0, 2), 11: (0, 5), 12: (0, 3), 13: (0, 2), 14: (0, 3), 15: (0, 3), 16: (0, 2), 17: (0, 1), 18: (0, 2), 19: (0, 3), 20: (0, 3), 21: (0, 1), 22: (0, 1), 23: (0, 3), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 2), 29: (0, 1), 30: (0, 1), 31: (0, 1), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 1), 37: (0, 1), 38: (0, 1), 39: (0, 1), 40: (0, 1), 41: (0, 1), 42: (0, 1), 43: (0, 0), 44: (0, 0)}, + + (2, 4): {0: (0, 8), 1: (0, 11), 2: (0, 5), 3: (0, 9), 4: (0, 4), 5: (0, 2), 6: (0, 6), 7: (0, 3), 8: (0, 8), 9: (0, 6), 10: (0, 3), 11: (0, 6), 12: (0, 4), 13: (0, 4), 14: (0, 4), 15: (0, 5), 16: (0, 3), 17: (0, 1), 18: (0, 3), 19: (0, 5), 20: (0, 4), 21: (0, 1), 22: (0, 2), 23: (0, 3), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 2), 29: (0, 1), 30: (0, 1), 31: (0, 1), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 1), 37: (0, 1), 38: (0, 2), 39: (0, 1), 40: (0, 1), 41: (0, 1), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0)}, + + (2, 5): {0: (0, 8), 1: (0, 11), 2: (0, 5), 3: (0, 9), 4: (0, 4), 5: (0, 2), 6: (0, 6), 7: (0, 3), 8: (0, 8), 9: (0, 7), 10: (0, 3), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 3), 17: (0, 1), 18: (0, 3), 19: (0, 5), 20: (0, 4), 21: (0, 1), 22: (0, 3), 23: (0, 3), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 2), 33: (0, 1), 34: (0, 1), 35: (0, 1), 36: (0, 2), 37: (0, 2), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0)}, + + (2, 6): {0: (0, 8), 1: (0, 11), 2: (0, 5), 3: (0, 9), 4: (0, 4), 5: (0, 3), 6: (0, 6), 7: (0, 3), 8: (0, 8), 9: (0, 7), 10: (0, 3), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 5), 20: (0, 4), 21: (0, 2), 22: (0, 3), 23: (0, 3), 24: (0, 1), 25: (0, 1), 26: (0, 1), 27: (0, 0), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 3), 33: (0, 1), 34: (0, 1), 35: (0, 2), 36: (0, 2), 37: (0, 2), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 0), 49: (0, 0)}, + + (2, 7): {0: (0, 8), 1: (0, 12), 2: (0, 5), 3: (0, 9), 4: (0, 4), 5: (0, 3), 6: (0, 6), 7: (0, 3), 8: (0, 8), 9: (0, 7), 10: (0, 3), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 6), 20: (0, 5), 21: (0, 2), 22: (0, 4), 23: (0, 3), 24: (0, 2), 25: (0, 2), 26: (0, 2), 27: (0, 0), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 3), 33: (0, 1), 34: (0, 1), 35: (0, 2), 36: (0, 2), 37: (0, 3), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 0), 49: (0, 0), 50: (0, 0), 51: (0, 0), 56: (0, 0), 57: (0, 0)}, + + (2, 8): {0: (0, 9), 1: (0, 12), 2: (0, 6), 3: (0, 11), 4: (0, 5), 5: (0, 3), 6: (0, 7), 7: (0, 3), 8: (0, 8), 9: (0, 7), 10: (0, 3), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 2), 36: (0, 2), 37: (0, 3), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 0), 57: (0, 0), 60: (0, 0), 61: (0, 0)}, + + (3, 0): {0: (0, 9), 1: (0, 12), 2: (0, 7), 3: (0, 11), 4: (0, 5), 5: (0, 3), 6: (0, 7), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 2), 36: (0, 2), 37: (0, 3), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 0), 57: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)}, + + (3, 1): {0: (0, 9), 1: (0, 13), 2: (0, 7), 3: (0, 12), 4: (0, 5), 5: (0, 3), 6: (0, 7), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 7), 12: (0, 4), 13: (0, 4), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 2), 36: (0, 2), 37: (0, 3), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 0), 57: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)}, + + (3, 2): {0: (0, 9), 1: (0, 13), 2: (0, 7), 3: (0, 12), 4: (0, 6), 5: (0, 3), 6: (0, 7), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 2), 30: (0, 2), 31: (0, 2), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 3), 36: (0, 2), 37: (0, 3), 38: (0, 2), 39: (0, 2), 40: (0, 2), 41: (0, 2), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 1), 57: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)}, + + (3, 3): {0: (0, 9), 1: (0, 13), 2: (0, 7), 3: (0, 12), 4: (0, 6), 5: (0, 3), 6: (0, 7), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 2), 57: (0, 1), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)}, + + (3, 4): {0: (0, 9), 1: (0, 13), 2: (0, 7), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 2), 57: (0, 1), 58: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)}, + + (3, 5): {0: (0, 9), 1: (0, 15), 2: (0, 8), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 3), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 3), 57: (0, 1), 58: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)}, + + (3, 6): {0: (0, 9), 1: (0, 15), 2: (0, 8), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 8), 9: (0, 8), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 4), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 3), 57: (0, 1), 58: (0, 0), 60: (0, 0), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0)}, + + (3, 7): {0: (0, 10), 1: (0, 16), 2: (0, 8), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 9), 9: (0, 9), 10: (0, 4), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 4), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 4), 23: (0, 4), 24: (0, 4), 25: (0, 3), 26: (0, 3), 27: (0, 1), 28: (0, 3), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 3), 57: (0, 1), 58: (0, 0), 60: (0, 1), 61: (0, 0), 65: (0, 0), 66: (0, 0), 67: (0, 0), 68: (0, 0)}, + + (3, 8): {0: (0, 11), 1: (0, 16), 2: (0, 8), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 9), 9: (0, 9), 10: (0, 5), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 5), 17: (0, 1), 18: (0, 3), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 5), 23: (0, 4), 24: (0, 5), 25: (0, 4), 26: (0, 4), 27: (0, 1), 28: (0, 4), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 3), 57: (0, 1), 58: (0, 0), 60: (0, 1), 61: (0, 0), 65: (0, 0), 66: (0, 1), 67: (0, 0), 68: (0, 0), 69: (0, 0)}, + + (3, 9): {0: (0, 11), 1: (0, 17), 2: (0, 9), 3: (0, 12), 4: (0, 7), 5: (0, 4), 6: (0, 8), 7: (0, 3), 8: (0, 9), 9: (0, 9), 10: (0, 6), 11: (0, 9), 12: (0, 4), 13: (0, 5), 14: (0, 5), 15: (0, 5), 16: (0, 5), 17: (0, 1), 18: (0, 4), 19: (0, 7), 20: (0, 6), 21: (0, 2), 22: (0, 5), 23: (0, 4), 24: (0, 5), 25: (0, 4), 26: (0, 4), 27: (0, 1), 28: (0, 4), 29: (0, 3), 30: (0, 3), 31: (0, 3), 32: (0, 4), 33: (0, 2), 34: (0, 2), 35: (0, 4), 36: (0, 2), 37: (0, 3), 38: (0, 3), 39: (0, 2), 40: (0, 2), 41: (0, 3), 42: (0, 2), 43: (0, 2), 44: (0, 1), 45: (0, 0), 46: (0, 0), 47: (0, 0), 48: (0, 1), 49: (0, 1), 50: (0, 0), 51: (0, 0), 56: (0, 3), 57: (0, 1), 58: (0, 0), 60: (0, 1), 61: (0, 0), 65: (0, 0), 66: (0, 1), 67: (0, 0), 68: (0, 0), 69: (0, 0)}, + +} diff --git a/kafka/protocol/commit.py b/kafka/protocol/commit.py index 31fc23707..a0439e7ef 100644 --- a/kafka/protocol/commit.py +++ b/kafka/protocol/commit.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from kafka.protocol.api import Request, Response -from kafka.protocol.types import Array, Int8, Int16, Int32, Int64, Schema, String +from kafka.protocol.types import Array, Int16, Int32, Int64, Schema, String class OffsetCommitResponse_v0(Response): @@ -41,6 +41,24 @@ class OffsetCommitResponse_v3(Response): ) +class OffsetCommitResponse_v4(Response): + API_KEY = 8 + API_VERSION = 4 + SCHEMA = OffsetCommitResponse_v3.SCHEMA + + +class OffsetCommitResponse_v5(Response): + API_KEY = 8 + API_VERSION = 5 + SCHEMA = OffsetCommitResponse_v4.SCHEMA + + +class OffsetCommitResponse_v6(Response): + API_KEY = 8 + API_VERSION = 6 + SCHEMA = OffsetCommitResponse_v5.SCHEMA + + class OffsetCommitRequest_v0(Request): API_KEY = 8 API_VERSION = 0 # Zookeeper-backed storage @@ -76,13 +94,13 @@ class OffsetCommitRequest_v1(Request): class OffsetCommitRequest_v2(Request): API_KEY = 8 - API_VERSION = 2 # added retention_time, dropped timestamp + API_VERSION = 2 RESPONSE_TYPE = OffsetCommitResponse_v2 SCHEMA = Schema( ('consumer_group', String('utf-8')), ('consumer_group_generation_id', Int32), ('consumer_id', String('utf-8')), - ('retention_time', Int64), + ('retention_time', Int64), # added retention_time, dropped timestamp ('topics', Array( ('topic', String('utf-8')), ('partitions', Array( @@ -90,7 +108,6 @@ class OffsetCommitRequest_v2(Request): ('offset', Int64), ('metadata', String('utf-8')))))) ) - DEFAULT_GENERATION_ID = -1 DEFAULT_RETENTION_TIME = -1 @@ -99,15 +116,63 @@ class OffsetCommitRequest_v3(Request): API_VERSION = 3 RESPONSE_TYPE = OffsetCommitResponse_v3 SCHEMA = OffsetCommitRequest_v2.SCHEMA + DEFAULT_RETENTION_TIME = -1 + + +class OffsetCommitRequest_v4(Request): + API_KEY = 8 + API_VERSION = 4 + RESPONSE_TYPE = OffsetCommitResponse_v4 + SCHEMA = OffsetCommitRequest_v3.SCHEMA + DEFAULT_RETENTION_TIME = -1 + + +class OffsetCommitRequest_v5(Request): + API_KEY = 8 + API_VERSION = 5 # drops retention_time + RESPONSE_TYPE = OffsetCommitResponse_v5 + SCHEMA = Schema( + ('consumer_group', String('utf-8')), + ('consumer_group_generation_id', Int32), + ('consumer_id', String('utf-8')), + ('topics', Array( + ('topic', String('utf-8')), + ('partitions', Array( + ('partition', Int32), + ('offset', Int64), + ('metadata', String('utf-8')))))) + ) + + +class OffsetCommitRequest_v6(Request): + API_KEY = 8 + API_VERSION = 6 + RESPONSE_TYPE = OffsetCommitResponse_v6 + SCHEMA = Schema( + ('consumer_group', String('utf-8')), + ('consumer_group_generation_id', Int32), + ('consumer_id', String('utf-8')), + ('topics', Array( + ('topic', String('utf-8')), + ('partitions', Array( + ('partition', Int32), + ('offset', Int64), + ('leader_epoch', Int32), # added for fencing / kip-320. default -1 + ('metadata', String('utf-8')))))) + ) OffsetCommitRequest = [ OffsetCommitRequest_v0, OffsetCommitRequest_v1, - OffsetCommitRequest_v2, OffsetCommitRequest_v3 + OffsetCommitRequest_v2, OffsetCommitRequest_v3, + OffsetCommitRequest_v4, OffsetCommitRequest_v5, + OffsetCommitRequest_v6, ] OffsetCommitResponse = [ OffsetCommitResponse_v0, OffsetCommitResponse_v1, - OffsetCommitResponse_v2, OffsetCommitResponse_v3 + OffsetCommitResponse_v2, OffsetCommitResponse_v3, + OffsetCommitResponse_v4, OffsetCommitResponse_v5, + OffsetCommitResponse_v6, ] @@ -163,6 +228,29 @@ class OffsetFetchResponse_v3(Response): ) +class OffsetFetchResponse_v4(Response): + API_KEY = 9 + API_VERSION = 4 + SCHEMA = OffsetFetchResponse_v3.SCHEMA + + +class OffsetFetchResponse_v5(Response): + API_KEY = 9 + API_VERSION = 5 + SCHEMA = Schema( + ('throttle_time_ms', Int32), + ('topics', Array( + ('topic', String('utf-8')), + ('partitions', Array( + ('partition', Int32), + ('offset', Int64), + ('leader_epoch', Int32), + ('metadata', String('utf-8')), + ('error_code', Int16))))), + ('error_code', Int16) + ) + + class OffsetFetchRequest_v0(Request): API_KEY = 9 API_VERSION = 0 # zookeeper-backed storage @@ -199,57 +287,27 @@ class OffsetFetchRequest_v3(Request): SCHEMA = OffsetFetchRequest_v2.SCHEMA +class OffsetFetchRequest_v4(Request): + API_KEY = 9 + API_VERSION = 4 + RESPONSE_TYPE = OffsetFetchResponse_v4 + SCHEMA = OffsetFetchRequest_v3.SCHEMA + + +class OffsetFetchRequest_v5(Request): + API_KEY = 9 + API_VERSION = 5 + RESPONSE_TYPE = OffsetFetchResponse_v5 + SCHEMA = OffsetFetchRequest_v4.SCHEMA + + OffsetFetchRequest = [ OffsetFetchRequest_v0, OffsetFetchRequest_v1, OffsetFetchRequest_v2, OffsetFetchRequest_v3, + OffsetFetchRequest_v4, OffsetFetchRequest_v5, ] OffsetFetchResponse = [ OffsetFetchResponse_v0, OffsetFetchResponse_v1, OffsetFetchResponse_v2, OffsetFetchResponse_v3, + OffsetFetchResponse_v4, OffsetFetchResponse_v5, ] - - -class GroupCoordinatorResponse_v0(Response): - API_KEY = 10 - API_VERSION = 0 - SCHEMA = Schema( - ('error_code', Int16), - ('coordinator_id', Int32), - ('host', String('utf-8')), - ('port', Int32) - ) - - -class GroupCoordinatorResponse_v1(Response): - API_KEY = 10 - API_VERSION = 1 - SCHEMA = Schema( - ('error_code', Int16), - ('error_message', String('utf-8')), - ('coordinator_id', Int32), - ('host', String('utf-8')), - ('port', Int32) - ) - - -class GroupCoordinatorRequest_v0(Request): - API_KEY = 10 - API_VERSION = 0 - RESPONSE_TYPE = GroupCoordinatorResponse_v0 - SCHEMA = Schema( - ('consumer_group', String('utf-8')) - ) - - -class GroupCoordinatorRequest_v1(Request): - API_KEY = 10 - API_VERSION = 1 - RESPONSE_TYPE = GroupCoordinatorResponse_v1 - SCHEMA = Schema( - ('coordinator_key', String('utf-8')), - ('coordinator_type', Int8) - ) - - -GroupCoordinatorRequest = [GroupCoordinatorRequest_v0, GroupCoordinatorRequest_v1] -GroupCoordinatorResponse = [GroupCoordinatorResponse_v0, GroupCoordinatorResponse_v1] diff --git a/kafka/protocol/fetch.py b/kafka/protocol/fetch.py index f367848ce..d193eafcf 100644 --- a/kafka/protocol/fetch.py +++ b/kafka/protocol/fetch.py @@ -14,7 +14,7 @@ class FetchResponse_v0(Response): ('partition', Int32), ('error_code', Int16), ('highwater_offset', Int64), - ('message_set', Bytes))))) + ('records', Bytes))))) ) @@ -29,7 +29,7 @@ class FetchResponse_v1(Response): ('partition', Int32), ('error_code', Int16), ('highwater_offset', Int64), - ('message_set', Bytes))))) + ('records', Bytes))))) ) @@ -46,6 +46,7 @@ class FetchResponse_v3(Response): class FetchResponse_v4(Response): + # Adds message format v2 API_KEY = 1 API_VERSION = 4 SCHEMA = Schema( @@ -60,7 +61,7 @@ class FetchResponse_v4(Response): ('aborted_transactions', Array( ('producer_id', Int64), ('first_offset', Int64))), - ('message_set', Bytes))))) + ('records', Bytes))))) ) @@ -80,7 +81,7 @@ class FetchResponse_v5(Response): ('aborted_transactions', Array( ('producer_id', Int64), ('first_offset', Int64))), - ('message_set', Bytes))))) + ('records', Bytes))))) ) @@ -115,7 +116,7 @@ class FetchResponse_v7(Response): ('aborted_transactions', Array( ('producer_id', Int64), ('first_offset', Int64))), - ('message_set', Bytes))))) + ('records', Bytes))))) ) @@ -156,7 +157,7 @@ class FetchResponse_v11(Response): ('producer_id', Int64), ('first_offset', Int64))), ('preferred_read_replica', Int32), - ('message_set', Bytes))))) + ('records', Bytes))))) ) @@ -211,6 +212,7 @@ class FetchRequest_v3(Request): class FetchRequest_v4(Request): # Adds isolation_level field + # Adds message format v2 API_KEY = 1 API_VERSION = 4 RESPONSE_TYPE = FetchResponse_v4 @@ -264,7 +266,7 @@ class FetchRequest_v6(Request): class FetchRequest_v7(Request): """ - Add incremental fetch requests + Add incremental fetch requests (see KIP-227) """ API_KEY = 1 API_VERSION = 7 @@ -285,7 +287,7 @@ class FetchRequest_v7(Request): ('log_start_offset', Int64), ('max_bytes', Int32))))), ('forgotten_topics_data', Array( - ('topic', String), + ('topic', String('utf-8')), ('partitions', Array(Int32)) )), ) @@ -325,7 +327,7 @@ class FetchRequest_v9(Request): ('log_start_offset', Int64), ('max_bytes', Int32))))), ('forgotten_topics_data', Array( - ('topic', String), + ('topic', String('utf-8')), ('partitions', Array(Int32)), )), ) @@ -365,7 +367,7 @@ class FetchRequest_v11(Request): ('log_start_offset', Int64), ('max_bytes', Int32))))), ('forgotten_topics_data', Array( - ('topic', String), + ('topic', String('utf-8')), ('partitions', Array(Int32)) )), ('rack_id', String('utf-8')), diff --git a/kafka/protocol/find_coordinator.py b/kafka/protocol/find_coordinator.py new file mode 100644 index 000000000..be5b45ded --- /dev/null +++ b/kafka/protocol/find_coordinator.py @@ -0,0 +1,64 @@ +from __future__ import absolute_import + +from kafka.protocol.api import Request, Response +from kafka.protocol.types import Int8, Int16, Int32, Schema, String + + +class FindCoordinatorResponse_v0(Response): + API_KEY = 10 + API_VERSION = 0 + SCHEMA = Schema( + ('error_code', Int16), + ('coordinator_id', Int32), + ('host', String('utf-8')), + ('port', Int32) + ) + + +class FindCoordinatorResponse_v1(Response): + API_KEY = 10 + API_VERSION = 1 + SCHEMA = Schema( + ('throttle_time_ms', Int32), + ('error_code', Int16), + ('error_message', String('utf-8')), + ('coordinator_id', Int32), + ('host', String('utf-8')), + ('port', Int32) + ) + + +class FindCoordinatorResponse_v2(Response): + API_KEY = 10 + API_VERSION = 2 + SCHEMA = FindCoordinatorResponse_v1.SCHEMA + + +class FindCoordinatorRequest_v0(Request): + API_KEY = 10 + API_VERSION = 0 + RESPONSE_TYPE = FindCoordinatorResponse_v0 + SCHEMA = Schema( + ('consumer_group', String('utf-8')) + ) + + +class FindCoordinatorRequest_v1(Request): + API_KEY = 10 + API_VERSION = 1 + RESPONSE_TYPE = FindCoordinatorResponse_v1 + SCHEMA = Schema( + ('coordinator_key', String('utf-8')), + ('coordinator_type', Int8) # 0: consumer, 1: transaction + ) + + +class FindCoordinatorRequest_v2(Request): + API_KEY = 10 + API_VERSION = 2 + RESPONSE_TYPE = FindCoordinatorResponse_v2 + SCHEMA = FindCoordinatorRequest_v1.SCHEMA + + +FindCoordinatorRequest = [FindCoordinatorRequest_v0, FindCoordinatorRequest_v1, FindCoordinatorRequest_v2] +FindCoordinatorResponse = [FindCoordinatorResponse_v0, FindCoordinatorResponse_v1, FindCoordinatorResponse_v2] diff --git a/kafka/protocol/group.py b/kafka/protocol/group.py index bcb96553b..3b32590ec 100644 --- a/kafka/protocol/group.py +++ b/kafka/protocol/group.py @@ -5,6 +5,10 @@ from kafka.protocol.types import Array, Bytes, Int16, Int32, Schema, String +DEFAULT_GENERATION_ID = -1 +UNKNOWN_MEMBER_ID = '' + + class JoinGroupResponse_v0(Response): API_KEY = 11 API_VERSION = 0 @@ -42,6 +46,12 @@ class JoinGroupResponse_v2(Response): ) +class JoinGroupResponse_v3(Response): + API_KEY = 11 + API_VERSION = 3 + SCHEMA = JoinGroupResponse_v2.SCHEMA + + class JoinGroupRequest_v0(Request): API_KEY = 11 API_VERSION = 0 @@ -55,7 +65,6 @@ class JoinGroupRequest_v0(Request): ('protocol_name', String('utf-8')), ('protocol_metadata', Bytes))) ) - UNKNOWN_MEMBER_ID = '' class JoinGroupRequest_v1(Request): @@ -72,7 +81,6 @@ class JoinGroupRequest_v1(Request): ('protocol_name', String('utf-8')), ('protocol_metadata', Bytes))) ) - UNKNOWN_MEMBER_ID = '' class JoinGroupRequest_v2(Request): @@ -80,14 +88,21 @@ class JoinGroupRequest_v2(Request): API_VERSION = 2 RESPONSE_TYPE = JoinGroupResponse_v2 SCHEMA = JoinGroupRequest_v1.SCHEMA + + +class JoinGroupRequest_v3(Request): + API_KEY = 11 + API_VERSION = 3 + RESPONSE_TYPE = JoinGroupResponse_v3 + SCHEMA = JoinGroupRequest_v2.SCHEMA UNKNOWN_MEMBER_ID = '' JoinGroupRequest = [ - JoinGroupRequest_v0, JoinGroupRequest_v1, JoinGroupRequest_v2 + JoinGroupRequest_v0, JoinGroupRequest_v1, JoinGroupRequest_v2, JoinGroupRequest_v3 ] JoinGroupResponse = [ - JoinGroupResponse_v0, JoinGroupResponse_v1, JoinGroupResponse_v2 + JoinGroupResponse_v0, JoinGroupResponse_v1, JoinGroupResponse_v2, JoinGroupResponse_v3 ] @@ -118,6 +133,12 @@ class SyncGroupResponse_v1(Response): ) +class SyncGroupResponse_v2(Response): + API_KEY = 14 + API_VERSION = 2 + SCHEMA = SyncGroupResponse_v1.SCHEMA + + class SyncGroupRequest_v0(Request): API_KEY = 14 API_VERSION = 0 @@ -139,8 +160,15 @@ class SyncGroupRequest_v1(Request): SCHEMA = SyncGroupRequest_v0.SCHEMA -SyncGroupRequest = [SyncGroupRequest_v0, SyncGroupRequest_v1] -SyncGroupResponse = [SyncGroupResponse_v0, SyncGroupResponse_v1] +class SyncGroupRequest_v2(Request): + API_KEY = 14 + API_VERSION = 2 + RESPONSE_TYPE = SyncGroupResponse_v2 + SCHEMA = SyncGroupRequest_v1.SCHEMA + + +SyncGroupRequest = [SyncGroupRequest_v0, SyncGroupRequest_v1, SyncGroupRequest_v2] +SyncGroupResponse = [SyncGroupResponse_v0, SyncGroupResponse_v1, SyncGroupResponse_v2] class MemberAssignment(Struct): @@ -170,6 +198,12 @@ class HeartbeatResponse_v1(Response): ) +class HeartbeatResponse_v2(Response): + API_KEY = 12 + API_VERSION = 2 + SCHEMA = HeartbeatResponse_v1.SCHEMA + + class HeartbeatRequest_v0(Request): API_KEY = 12 API_VERSION = 0 @@ -188,8 +222,15 @@ class HeartbeatRequest_v1(Request): SCHEMA = HeartbeatRequest_v0.SCHEMA -HeartbeatRequest = [HeartbeatRequest_v0, HeartbeatRequest_v1] -HeartbeatResponse = [HeartbeatResponse_v0, HeartbeatResponse_v1] +class HeartbeatRequest_v2(Request): + API_KEY = 12 + API_VERSION = 2 + RESPONSE_TYPE = HeartbeatResponse_v2 + SCHEMA = HeartbeatRequest_v1.SCHEMA + + +HeartbeatRequest = [HeartbeatRequest_v0, HeartbeatRequest_v1, HeartbeatRequest_v2] +HeartbeatResponse = [HeartbeatResponse_v0, HeartbeatResponse_v1, HeartbeatResponse_v2] class LeaveGroupResponse_v0(Response): @@ -209,6 +250,12 @@ class LeaveGroupResponse_v1(Response): ) +class LeaveGroupResponse_v2(Response): + API_KEY = 13 + API_VERSION = 2 + SCHEMA = LeaveGroupResponse_v1.SCHEMA + + class LeaveGroupRequest_v0(Request): API_KEY = 13 API_VERSION = 0 @@ -226,5 +273,12 @@ class LeaveGroupRequest_v1(Request): SCHEMA = LeaveGroupRequest_v0.SCHEMA -LeaveGroupRequest = [LeaveGroupRequest_v0, LeaveGroupRequest_v1] -LeaveGroupResponse = [LeaveGroupResponse_v0, LeaveGroupResponse_v1] +class LeaveGroupRequest_v2(Request): + API_KEY = 13 + API_VERSION = 2 + RESPONSE_TYPE = LeaveGroupResponse_v2 + SCHEMA = LeaveGroupRequest_v1.SCHEMA + + +LeaveGroupRequest = [LeaveGroupRequest_v0, LeaveGroupRequest_v1, LeaveGroupRequest_v2] +LeaveGroupResponse = [LeaveGroupResponse_v0, LeaveGroupResponse_v1, LeaveGroupResponse_v2] diff --git a/kafka/protocol/offset.py b/kafka/protocol/list_offsets.py similarity index 73% rename from kafka/protocol/offset.py rename to kafka/protocol/list_offsets.py index 1ed382b0d..2e36dd660 100644 --- a/kafka/protocol/offset.py +++ b/kafka/protocol/list_offsets.py @@ -12,7 +12,7 @@ class OffsetResetStrategy(object): NONE = 0 -class OffsetResponse_v0(Response): +class ListOffsetsResponse_v0(Response): API_KEY = 2 API_VERSION = 0 SCHEMA = Schema( @@ -24,7 +24,7 @@ class OffsetResponse_v0(Response): ('offsets', Array(Int64)))))) ) -class OffsetResponse_v1(Response): +class ListOffsetsResponse_v1(Response): API_KEY = 2 API_VERSION = 1 SCHEMA = Schema( @@ -38,7 +38,7 @@ class OffsetResponse_v1(Response): ) -class OffsetResponse_v2(Response): +class ListOffsetsResponse_v2(Response): API_KEY = 2 API_VERSION = 2 SCHEMA = Schema( @@ -53,16 +53,16 @@ class OffsetResponse_v2(Response): ) -class OffsetResponse_v3(Response): +class ListOffsetsResponse_v3(Response): """ on quota violation, brokers send out responses before throttling """ API_KEY = 2 API_VERSION = 3 - SCHEMA = OffsetResponse_v2.SCHEMA + SCHEMA = ListOffsetsResponse_v2.SCHEMA -class OffsetResponse_v4(Response): +class ListOffsetsResponse_v4(Response): """ Add leader_epoch to response """ @@ -81,19 +81,19 @@ class OffsetResponse_v4(Response): ) -class OffsetResponse_v5(Response): +class ListOffsetsResponse_v5(Response): """ adds a new error code, OFFSET_NOT_AVAILABLE """ API_KEY = 2 API_VERSION = 5 - SCHEMA = OffsetResponse_v4.SCHEMA + SCHEMA = ListOffsetsResponse_v4.SCHEMA -class OffsetRequest_v0(Request): +class ListOffsetsRequest_v0(Request): API_KEY = 2 API_VERSION = 0 - RESPONSE_TYPE = OffsetResponse_v0 + RESPONSE_TYPE = ListOffsetsResponse_v0 SCHEMA = Schema( ('replica_id', Int32), ('topics', Array( @@ -107,10 +107,10 @@ class OffsetRequest_v0(Request): 'replica_id': -1 } -class OffsetRequest_v1(Request): +class ListOffsetsRequest_v1(Request): API_KEY = 2 API_VERSION = 1 - RESPONSE_TYPE = OffsetResponse_v1 + RESPONSE_TYPE = ListOffsetsResponse_v1 SCHEMA = Schema( ('replica_id', Int32), ('topics', Array( @@ -124,10 +124,10 @@ class OffsetRequest_v1(Request): } -class OffsetRequest_v2(Request): +class ListOffsetsRequest_v2(Request): API_KEY = 2 API_VERSION = 2 - RESPONSE_TYPE = OffsetResponse_v2 + RESPONSE_TYPE = ListOffsetsResponse_v2 SCHEMA = Schema( ('replica_id', Int32), ('isolation_level', Int8), # <- added isolation_level @@ -142,23 +142,23 @@ class OffsetRequest_v2(Request): } -class OffsetRequest_v3(Request): +class ListOffsetsRequest_v3(Request): API_KEY = 2 API_VERSION = 3 - RESPONSE_TYPE = OffsetResponse_v3 - SCHEMA = OffsetRequest_v2.SCHEMA + RESPONSE_TYPE = ListOffsetsResponse_v3 + SCHEMA = ListOffsetsRequest_v2.SCHEMA DEFAULTS = { 'replica_id': -1 } -class OffsetRequest_v4(Request): +class ListOffsetsRequest_v4(Request): """ Add current_leader_epoch to request """ API_KEY = 2 API_VERSION = 4 - RESPONSE_TYPE = OffsetResponse_v4 + RESPONSE_TYPE = ListOffsetsResponse_v4 SCHEMA = Schema( ('replica_id', Int32), ('isolation_level', Int8), # <- added isolation_level @@ -166,7 +166,7 @@ class OffsetRequest_v4(Request): ('topic', String('utf-8')), ('partitions', Array( ('partition', Int32), - ('current_leader_epoch', Int64), + ('current_leader_epoch', Int32), ('timestamp', Int64))))) ) DEFAULTS = { @@ -174,21 +174,21 @@ class OffsetRequest_v4(Request): } -class OffsetRequest_v5(Request): +class ListOffsetsRequest_v5(Request): API_KEY = 2 API_VERSION = 5 - RESPONSE_TYPE = OffsetResponse_v5 - SCHEMA = OffsetRequest_v4.SCHEMA + RESPONSE_TYPE = ListOffsetsResponse_v5 + SCHEMA = ListOffsetsRequest_v4.SCHEMA DEFAULTS = { 'replica_id': -1 } -OffsetRequest = [ - OffsetRequest_v0, OffsetRequest_v1, OffsetRequest_v2, - OffsetRequest_v3, OffsetRequest_v4, OffsetRequest_v5, +ListOffsetsRequest = [ + ListOffsetsRequest_v0, ListOffsetsRequest_v1, ListOffsetsRequest_v2, + ListOffsetsRequest_v3, ListOffsetsRequest_v4, ListOffsetsRequest_v5, ] -OffsetResponse = [ - OffsetResponse_v0, OffsetResponse_v1, OffsetResponse_v2, - OffsetResponse_v3, OffsetResponse_v4, OffsetResponse_v5, +ListOffsetsResponse = [ + ListOffsetsResponse_v0, ListOffsetsResponse_v1, ListOffsetsResponse_v2, + ListOffsetsResponse_v3, ListOffsetsResponse_v4, ListOffsetsResponse_v5, ] diff --git a/kafka/protocol/metadata.py b/kafka/protocol/metadata.py index 414e5b84a..3291be82d 100644 --- a/kafka/protocol/metadata.py +++ b/kafka/protocol/metadata.py @@ -128,6 +128,42 @@ class MetadataResponse_v5(Response): ) +class MetadataResponse_v6(Response): + """Metadata Request/Response v6 is the same as v5, + but on quota violation, brokers send out responses before throttling.""" + API_KEY = 3 + API_VERSION = 6 + SCHEMA = MetadataResponse_v5.SCHEMA + + +class MetadataResponse_v7(Response): + """v7 adds per-partition leader_epoch field""" + API_KEY = 3 + API_VERSION = 7 + SCHEMA = Schema( + ('throttle_time_ms', Int32), + ('brokers', Array( + ('node_id', Int32), + ('host', String('utf-8')), + ('port', Int32), + ('rack', String('utf-8')))), + ('cluster_id', String('utf-8')), + ('controller_id', Int32), + ('topics', Array( + ('error_code', Int16), + ('topic', String('utf-8')), + ('is_internal', Boolean), + ('partitions', Array( + ('error_code', Int16), + ('partition', Int32), + ('leader', Int32), + ('leader_epoch', Int32), + ('replicas', Array(Int32)), + ('isr', Array(Int32)), + ('offline_replicas', Array(Int32)))))) + ) + + class MetadataRequest_v0(Request): API_KEY = 3 API_VERSION = 0 @@ -135,7 +171,7 @@ class MetadataRequest_v0(Request): SCHEMA = Schema( ('topics', Array(String('utf-8'))) ) - ALL_TOPICS = None # Empty Array (len 0) for topics returns all topics + ALL_TOPICS = [] # Empty Array (len 0) for topics returns all topics class MetadataRequest_v1(Request): @@ -143,8 +179,8 @@ class MetadataRequest_v1(Request): API_VERSION = 1 RESPONSE_TYPE = MetadataResponse_v1 SCHEMA = MetadataRequest_v0.SCHEMA - ALL_TOPICS = -1 # Null Array (len -1) for topics returns all topics - NO_TOPICS = None # Empty array (len 0) for topics returns no topics + ALL_TOPICS = None # Null Array (len -1) for topics returns all topics + NO_TOPICS = [] # Empty array (len 0) for topics returns no topics class MetadataRequest_v2(Request): @@ -152,8 +188,8 @@ class MetadataRequest_v2(Request): API_VERSION = 2 RESPONSE_TYPE = MetadataResponse_v2 SCHEMA = MetadataRequest_v1.SCHEMA - ALL_TOPICS = -1 # Null Array (len -1) for topics returns all topics - NO_TOPICS = None # Empty array (len 0) for topics returns no topics + ALL_TOPICS = None + NO_TOPICS = [] class MetadataRequest_v3(Request): @@ -161,8 +197,8 @@ class MetadataRequest_v3(Request): API_VERSION = 3 RESPONSE_TYPE = MetadataResponse_v3 SCHEMA = MetadataRequest_v1.SCHEMA - ALL_TOPICS = -1 # Null Array (len -1) for topics returns all topics - NO_TOPICS = None # Empty array (len 0) for topics returns no topics + ALL_TOPICS = None + NO_TOPICS = [] class MetadataRequest_v4(Request): @@ -173,8 +209,8 @@ class MetadataRequest_v4(Request): ('topics', Array(String('utf-8'))), ('allow_auto_topic_creation', Boolean) ) - ALL_TOPICS = -1 # Null Array (len -1) for topics returns all topics - NO_TOPICS = None # Empty array (len 0) for topics returns no topics + ALL_TOPICS = None + NO_TOPICS = [] class MetadataRequest_v5(Request): @@ -186,15 +222,35 @@ class MetadataRequest_v5(Request): API_VERSION = 5 RESPONSE_TYPE = MetadataResponse_v5 SCHEMA = MetadataRequest_v4.SCHEMA - ALL_TOPICS = -1 # Null Array (len -1) for topics returns all topics - NO_TOPICS = None # Empty array (len 0) for topics returns no topics + ALL_TOPICS = None + NO_TOPICS = [] + + +class MetadataRequest_v6(Request): + API_KEY = 3 + API_VERSION = 6 + RESPONSE_TYPE = MetadataResponse_v6 + SCHEMA = MetadataRequest_v5.SCHEMA + ALL_TOPICS = None + NO_TOPICS = [] + + +class MetadataRequest_v7(Request): + API_KEY = 3 + API_VERSION = 7 + RESPONSE_TYPE = MetadataResponse_v7 + SCHEMA = MetadataRequest_v6.SCHEMA + ALL_TOPICS = None + NO_TOPICS = [] MetadataRequest = [ MetadataRequest_v0, MetadataRequest_v1, MetadataRequest_v2, - MetadataRequest_v3, MetadataRequest_v4, MetadataRequest_v5 + MetadataRequest_v3, MetadataRequest_v4, MetadataRequest_v5, + MetadataRequest_v6, MetadataRequest_v7, ] MetadataResponse = [ MetadataResponse_v0, MetadataResponse_v1, MetadataResponse_v2, - MetadataResponse_v3, MetadataResponse_v4, MetadataResponse_v5 + MetadataResponse_v3, MetadataResponse_v4, MetadataResponse_v5, + MetadataResponse_v6, MetadataResponse_v7, ] diff --git a/kafka/protocol/offset_for_leader_epoch.py b/kafka/protocol/offset_for_leader_epoch.py new file mode 100644 index 000000000..8465588a3 --- /dev/null +++ b/kafka/protocol/offset_for_leader_epoch.py @@ -0,0 +1,140 @@ +from __future__ import absolute_import + +from kafka.protocol.api import Request, Response +from kafka.protocol.types import Array, CompactArray, CompactString, Int16, Int32, Int64, Schema, String, TaggedFields + + +class OffsetForLeaderEpochResponse_v0(Response): + API_KEY = 23 + API_VERSION = 0 + SCHEMA = Schema( + ('topics', Array( + ('topic', String('utf-8')), + ('partitions', Array( + ('error_code', Int16), + ('partition', Int32), + ('end_offset', Int64)))))) + + +class OffsetForLeaderEpochResponse_v1(Response): + API_KEY = 23 + API_VERSION = 1 + SCHEMA = Schema( + ('topics', Array( + ('topic', String('utf-8')), + ('partitions', Array( + ('error_code', Int16), + ('partition', Int32), + ('leader_epoch', Int32), + ('end_offset', Int64)))))) + + +class OffsetForLeaderEpochResponse_v2(Response): + API_KEY = 23 + API_VERSION = 2 + SCHEMA = Schema( + ('throttle_time_ms', Int32), + ('topics', Array( + ('topic', String('utf-8')), + ('partitions', Array( + ('error_code', Int16), + ('partition', Int32), + ('leader_epoch', Int32), + ('end_offset', Int64)))))) + + +class OffsetForLeaderEpochResponse_v3(Response): + API_KEY = 23 + API_VERSION = 3 + SCHEMA = OffsetForLeaderEpochResponse_v2.SCHEMA + + +class OffsetForLeaderEpochResponse_v4(Response): + API_KEY = 23 + API_VERSION = 4 + SCHEMA = Schema( + ('throttle_time_ms', Int32), + ('topics', CompactArray( + ('topic', CompactString('utf-8')), + ('partitions', CompactArray( + ('error_code', Int16), + ('partition', Int32), + ('leader_epoch', Int32), + ('end_offset', Int64), + ('tags', TaggedFields))), + ('tags', TaggedFields))), + ('tags', TaggedFields)) + + +class OffsetForLeaderEpochRequest_v0(Request): + API_KEY = 23 + API_VERSION = 0 + RESPONSE_TYPE = OffsetForLeaderEpochResponse_v0 + SCHEMA = Schema( + ('topics', Array( + ('topic', String('utf-8')), + ('partitions', Array( + ('partition', Int32), + ('leader_epoch', Int32)))))) + + +class OffsetForLeaderEpochRequest_v1(Request): + API_KEY = 23 + API_VERSION = 1 + RESPONSE_TYPE = OffsetForLeaderEpochResponse_v1 + SCHEMA = OffsetForLeaderEpochRequest_v0.SCHEMA + + +class OffsetForLeaderEpochRequest_v2(Request): + API_KEY = 23 + API_VERSION = 2 + RESPONSE_TYPE = OffsetForLeaderEpochResponse_v2 + SCHEMA = Schema( + ('topics', Array( + ('topic', String('utf-8')), + ('partitions', Array( + ('partition', Int32), + ('current_leader_epoch', Int32), + ('leader_epoch', Int32)))))) + + +class OffsetForLeaderEpochRequest_v3(Request): + API_KEY = 23 + API_VERSION = 3 + RESPONSE_TYPE = OffsetForLeaderEpochResponse_v3 + SCHEMA = Schema( + ('replica_id', Int32), + ('topics', Array( + ('topic', String('utf-8')), + ('partitions', Array( + ('partition', Int32), + ('current_leader_epoch', Int32), + ('leader_epoch', Int32)))))) + + +class OffsetForLeaderEpochRequest_v4(Request): + API_KEY = 23 + API_VERSION = 4 + RESPONSE_TYPE = OffsetForLeaderEpochResponse_v4 + SCHEMA = Schema( + ('replica_id', Int32), + ('topics', CompactArray( + ('topic', CompactString('utf-8')), + ('partitions', CompactArray( + ('partition', Int32), + ('current_leader_epoch', Int32), + ('leader_epoch', Int32), + ('tags', TaggedFields))), + ('tags', TaggedFields))), + ('tags', TaggedFields)) + +OffsetForLeaderEpochRequest = [ + OffsetForLeaderEpochRequest_v0, OffsetForLeaderEpochRequest_v1, + OffsetForLeaderEpochRequest_v2, OffsetForLeaderEpochRequest_v3, + OffsetForLeaderEpochRequest_v4, +] +OffsetForLeaderEpochResponse = [ + OffsetForLeaderEpochResponse_v0, OffsetForLeaderEpochResponse_v1, + OffsetForLeaderEpochResponse_v2, OffsetForLeaderEpochResponse_v3, + OffsetForLeaderEpochResponse_v4, +] diff --git a/kafka/protocol/parser.py b/kafka/protocol/parser.py index a9e767220..e7799fce6 100644 --- a/kafka/protocol/parser.py +++ b/kafka/protocol/parser.py @@ -4,7 +4,7 @@ import logging import kafka.errors as Errors -from kafka.protocol.commit import GroupCoordinatorResponse +from kafka.protocol.find_coordinator import FindCoordinatorResponse from kafka.protocol.frame import KafkaBytes from kafka.protocol.types import Int32, TaggedFields from kafka.version import __version__ @@ -142,7 +142,7 @@ def _process_response(self, read_buffer): # 0.8.2 quirk if (recv_correlation_id == 0 and correlation_id != 0 and - request.RESPONSE_TYPE is GroupCoordinatorResponse[0] and + request.RESPONSE_TYPE is FindCoordinatorResponse[0] and (self._api_version == (0, 8, 2) or self._api_version is None)): log.warning('Kafka 0.8.2 quirk -- GroupCoordinatorResponse' ' Correlation ID does not match request. This' diff --git a/kafka/protocol/produce.py b/kafka/protocol/produce.py index 9b3f6bf55..3076a2810 100644 --- a/kafka/protocol/produce.py +++ b/kafka/protocol/produce.py @@ -47,6 +47,7 @@ class ProduceResponse_v2(Response): class ProduceResponse_v3(Response): + # Adds support for message format v2 API_KEY = 0 API_VERSION = 3 SCHEMA = ProduceResponse_v2.SCHEMA @@ -141,7 +142,7 @@ class ProduceRequest_v0(ProduceRequest): ('topic', String('utf-8')), ('partitions', Array( ('partition', Int32), - ('messages', Bytes))))) + ('records', Bytes))))) ) @@ -158,6 +159,7 @@ class ProduceRequest_v2(ProduceRequest): class ProduceRequest_v3(ProduceRequest): + # Adds support for message format v2 API_VERSION = 3 RESPONSE_TYPE = ProduceResponse_v3 SCHEMA = Schema( @@ -168,7 +170,7 @@ class ProduceRequest_v3(ProduceRequest): ('topic', String('utf-8')), ('partitions', Array( ('partition', Int32), - ('messages', Bytes))))) + ('records', Bytes))))) ) diff --git a/kafka/protocol/sasl_authenticate.py b/kafka/protocol/sasl_authenticate.py new file mode 100644 index 000000000..a2b9b1988 --- /dev/null +++ b/kafka/protocol/sasl_authenticate.py @@ -0,0 +1,42 @@ +from __future__ import absolute_import + +from kafka.protocol.api import Request, Response +from kafka.protocol.types import Bytes, Int16, Int64, Schema, String + + +class SaslAuthenticateResponse_v0(Response): + API_KEY = 36 + API_VERSION = 0 + SCHEMA = Schema( + ('error_code', Int16), + ('error_message', String('utf-8')), + ('auth_bytes', Bytes)) + + +class SaslAuthenticateResponse_v1(Response): + API_KEY = 36 + API_VERSION = 1 + SCHEMA = Schema( + ('error_code', Int16), + ('error_message', String('utf-8')), + ('auth_bytes', Bytes), + ('session_lifetime_ms', Int64)) + + +class SaslAuthenticateRequest_v0(Request): + API_KEY = 36 + API_VERSION = 0 + RESPONSE_TYPE = SaslAuthenticateResponse_v0 + SCHEMA = Schema( + ('auth_bytes', Bytes)) + + +class SaslAuthenticateRequest_v1(Request): + API_KEY = 36 + API_VERSION = 1 + RESPONSE_TYPE = SaslAuthenticateResponse_v1 + SCHEMA = SaslAuthenticateRequest_v0.SCHEMA + + +SaslAuthenticateRequest = [SaslAuthenticateRequest_v0, SaslAuthenticateRequest_v1] +SaslAuthenticateResponse = [SaslAuthenticateResponse_v0, SaslAuthenticateResponse_v1] diff --git a/kafka/protocol/sasl_handshake.py b/kafka/protocol/sasl_handshake.py new file mode 100644 index 000000000..e91c856ca --- /dev/null +++ b/kafka/protocol/sasl_handshake.py @@ -0,0 +1,39 @@ +from __future__ import absolute_import + +from kafka.protocol.api import Request, Response +from kafka.protocol.types import Array, Int16, Schema, String + + +class SaslHandshakeResponse_v0(Response): + API_KEY = 17 + API_VERSION = 0 + SCHEMA = Schema( + ('error_code', Int16), + ('enabled_mechanisms', Array(String('utf-8'))) + ) + + +class SaslHandshakeResponse_v1(Response): + API_KEY = 17 + API_VERSION = 1 + SCHEMA = SaslHandshakeResponse_v0.SCHEMA + + +class SaslHandshakeRequest_v0(Request): + API_KEY = 17 + API_VERSION = 0 + RESPONSE_TYPE = SaslHandshakeResponse_v0 + SCHEMA = Schema( + ('mechanism', String('utf-8')) + ) + + +class SaslHandshakeRequest_v1(Request): + API_KEY = 17 + API_VERSION = 1 + RESPONSE_TYPE = SaslHandshakeResponse_v1 + SCHEMA = SaslHandshakeRequest_v0.SCHEMA + + +SaslHandshakeRequest = [SaslHandshakeRequest_v0, SaslHandshakeRequest_v1] +SaslHandshakeResponse = [SaslHandshakeResponse_v0, SaslHandshakeResponse_v1] diff --git a/kafka/record/default_records.py b/kafka/record/default_records.py index a098c42a9..14732cb06 100644 --- a/kafka/record/default_records.py +++ b/kafka/record/default_records.py @@ -136,6 +136,10 @@ def __init__(self, buffer): def base_offset(self): return self._header_data[0] + @property + def leader_epoch(self): + return self._header_data[2] + @property def magic(self): return self._header_data[3] @@ -269,8 +273,12 @@ def _read_msg( "payload, but instead read {}".format(length, pos - start_pos)) self._pos = pos - return DefaultRecord( - offset, timestamp, self.timestamp_type, key, value, headers) + if self.is_control_batch: + return ControlRecord( + offset, timestamp, self.timestamp_type, key, value, headers) + else: + return DefaultRecord( + offset, timestamp, self.timestamp_type, key, value, headers) def __iter__(self): self._maybe_uncompress() @@ -362,6 +370,45 @@ def __repr__(self): ) +class ControlRecord(DefaultRecord): + __slots__ = ("_offset", "_timestamp", "_timestamp_type", "_key", "_value", + "_headers", "_version", "_type") + + KEY_STRUCT = struct.Struct( + ">h" # Current Version => Int16 + "h" # Type => Int16 (0 indicates an abort marker, 1 indicates a commit) + ) + + def __init__(self, offset, timestamp, timestamp_type, key, value, headers): + super(ControlRecord, self).__init__(offset, timestamp, timestamp_type, key, value, headers) + (self._version, self._type) = self.KEY_STRUCT.unpack(self._key) + + # see https://kafka.apache.org/documentation/#controlbatch + @property + def version(self): + return self._version + + @property + def type(self): + return self._type + + @property + def abort(self): + return self._type == 0 + + @property + def commit(self): + return self._type == 1 + + def __repr__(self): + return ( + "ControlRecord(offset={!r}, timestamp={!r}, timestamp_type={!r}," + " version={!r}, type={!r} <{!s}>)".format( + self._offset, self._timestamp, self._timestamp_type, + self._version, self._type, "abort" if self.abort else "commit") + ) + + class DefaultRecordBatchBuilder(DefaultRecordBase, ABCRecordBatchBuilder): # excluding key, value and headers: diff --git a/kafka/sasl/__init__.py b/kafka/sasl/__init__.py new file mode 100644 index 000000000..90f05e733 --- /dev/null +++ b/kafka/sasl/__init__.py @@ -0,0 +1,34 @@ +from __future__ import absolute_import + +import platform + +from kafka.sasl.gssapi import SaslMechanismGSSAPI +from kafka.sasl.msk import SaslMechanismAwsMskIam +from kafka.sasl.oauth import SaslMechanismOAuth +from kafka.sasl.plain import SaslMechanismPlain +from kafka.sasl.scram import SaslMechanismScram +from kafka.sasl.sspi import SaslMechanismSSPI + + +SASL_MECHANISMS = {} + + +def register_sasl_mechanism(name, klass, overwrite=False): + if not overwrite and name in SASL_MECHANISMS: + raise ValueError('Sasl mechanism %s already defined!' % name) + SASL_MECHANISMS[name] = klass + + +def get_sasl_mechanism(name): + return SASL_MECHANISMS[name] + + +register_sasl_mechanism('AWS_MSK_IAM', SaslMechanismAwsMskIam) +if platform.system() == 'Windows': + register_sasl_mechanism('GSSAPI', SaslMechanismSSPI) +else: + register_sasl_mechanism('GSSAPI', SaslMechanismGSSAPI) +register_sasl_mechanism('OAUTHBEARER', SaslMechanismOAuth) +register_sasl_mechanism('PLAIN', SaslMechanismPlain) +register_sasl_mechanism('SCRAM-SHA-256', SaslMechanismScram) +register_sasl_mechanism('SCRAM-SHA-512', SaslMechanismScram) diff --git a/kafka/sasl/abc.py b/kafka/sasl/abc.py new file mode 100644 index 000000000..8977c7c23 --- /dev/null +++ b/kafka/sasl/abc.py @@ -0,0 +1,32 @@ +from __future__ import absolute_import + +import abc + + +class SaslMechanism(object): + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def __init__(self, **config): + pass + + @abc.abstractmethod + def auth_bytes(self): + pass + + @abc.abstractmethod + def receive(self, auth_bytes): + pass + + @abc.abstractmethod + def is_done(self): + pass + + @abc.abstractmethod + def is_authenticated(self): + pass + + def auth_details(self): + if not self.is_authenticated: + raise RuntimeError('Not authenticated yet!') + return 'Authenticated via SASL' diff --git a/kafka/sasl/gssapi.py b/kafka/sasl/gssapi.py new file mode 100644 index 000000000..be84269da --- /dev/null +++ b/kafka/sasl/gssapi.py @@ -0,0 +1,87 @@ +from __future__ import absolute_import + +# needed for SASL_GSSAPI authentication: +try: + import gssapi + from gssapi.raw.misc import GSSError +except (ImportError, OSError): + #no gssapi available, will disable gssapi mechanism + gssapi = None + GSSError = None + +from kafka.sasl.abc import SaslMechanism + + +class SaslMechanismGSSAPI(SaslMechanism): + # Establish security context and negotiate protection level + # For reference RFC 2222, section 7.2.1 + + SASL_QOP_AUTH = 1 + SASL_QOP_AUTH_INT = 2 + SASL_QOP_AUTH_CONF = 4 + + def __init__(self, **config): + assert gssapi is not None, 'GSSAPI lib not available' + if 'sasl_kerberos_name' not in config and 'sasl_kerberos_service_name' not in config: + raise ValueError('sasl_kerberos_service_name or sasl_kerberos_name required for GSSAPI sasl configuration') + self._is_done = False + self._is_authenticated = False + if config.get('sasl_kerberos_name', None) is not None: + self.auth_id = str(config['sasl_kerberos_name']) + else: + kerberos_domain_name = config.get('sasl_kerberos_domain_name', '') or config.get('host', '') + self.auth_id = config['sasl_kerberos_service_name'] + '@' + kerberos_domain_name + if isinstance(config.get('sasl_kerberos_name', None), gssapi.Name): + self.gssapi_name = config['sasl_kerberos_name'] + else: + self.gssapi_name = gssapi.Name(self.auth_id, name_type=gssapi.NameType.hostbased_service).canonicalize(gssapi.MechType.kerberos) + self._client_ctx = gssapi.SecurityContext(name=self.gssapi_name, usage='initiate') + self._next_token = self._client_ctx.step(None) + + def auth_bytes(self): + # GSSAPI Auth does not have a final broker->client message + # so mark is_done after the final auth_bytes are provided + # in practice we'll still receive a response when using SaslAuthenticate + # but not when using the prior unframed approach. + if self._client_ctx.complete: + self._is_done = True + self._is_authenticated = True + return self._next_token or b'' + + def receive(self, auth_bytes): + if not self._client_ctx.complete: + # The server will send a token back. Processing of this token either + # establishes a security context, or it needs further token exchange. + # The gssapi will be able to identify the needed next step. + self._next_token = self._client_ctx.step(auth_bytes) + elif self._is_done: + # The final step of gssapi is send, so we do not expect any additional bytes + # however, allow an empty message to support SaslAuthenticate response + if auth_bytes != b'': + raise ValueError("Unexpected receive auth_bytes after sasl/gssapi completion") + else: + # unwraps message containing supported protection levels and msg size + msg = self._client_ctx.unwrap(auth_bytes).message + # Kafka currently doesn't support integrity or confidentiality security layers, so we + # simply set QoP to 'auth' only (first octet). We reuse the max message size proposed + # by the server + client_flags = self.SASL_QOP_AUTH + server_flags = msg[0] + message_parts = [ + bytes(client_flags & server_flags), + msg[:1], + self.auth_id.encode('utf-8'), + ] + # add authorization identity to the response, and GSS-wrap + self._next_token = self._client_ctx.wrap(b''.join(message_parts), False).message + + def is_done(self): + return self._is_done + + def is_authenticated(self): + return self._is_authenticated + + def auth_details(self): + if not self.is_authenticated: + raise RuntimeError('Not authenticated yet!') + return 'Authenticated as %s to %s via SASL / GSSAPI' % (self._client_ctx.initiator_name, self._client_ctx.target_name) diff --git a/kafka/sasl/msk.py b/kafka/sasl/msk.py new file mode 100644 index 000000000..db56b4801 --- /dev/null +++ b/kafka/sasl/msk.py @@ -0,0 +1,233 @@ +from __future__ import absolute_import + +import datetime +import hashlib +import hmac +import json +import string + +# needed for AWS_MSK_IAM authentication: +try: + from botocore.session import Session as BotoSession +except ImportError: + # no botocore available, will disable AWS_MSK_IAM mechanism + BotoSession = None + +from kafka.sasl.abc import SaslMechanism +from kafka.vendor.six.moves import urllib + + +class SaslMechanismAwsMskIam(SaslMechanism): + def __init__(self, **config): + assert BotoSession is not None, 'AWS_MSK_IAM requires the "botocore" package' + assert config.get('security_protocol', '') == 'SASL_SSL', 'AWS_MSK_IAM requires SASL_SSL' + assert 'host' in config, 'AWS_MSK_IAM requires host configuration' + self.host = config['host'] + self._auth = None + self._is_done = False + self._is_authenticated = False + + def auth_bytes(self): + session = BotoSession() + credentials = session.get_credentials().get_frozen_credentials() + client = AwsMskIamClient( + host=self.host, + access_key=credentials.access_key, + secret_key=credentials.secret_key, + region=session.get_config_variable('region'), + token=credentials.token, + ) + return client.first_message() + + def receive(self, auth_bytes): + self._is_done = True + self._is_authenticated = auth_bytes != b'' + self._auth = auth_bytes.deode('utf-8') + + def is_done(self): + return self._is_done + + def is_authenticated(self): + return self._is_authenticated + + def auth_details(self): + if not self.is_authenticated: + raise RuntimeError('Not authenticated yet!') + return 'Authenticated via SASL / AWS_MSK_IAM %s' % (self._auth,) + + +class AwsMskIamClient: + UNRESERVED_CHARS = string.ascii_letters + string.digits + '-._~' + + def __init__(self, host, access_key, secret_key, region, token=None): + """ + Arguments: + host (str): The hostname of the broker. + access_key (str): An AWS_ACCESS_KEY_ID. + secret_key (str): An AWS_SECRET_ACCESS_KEY. + region (str): An AWS_REGION. + token (Optional[str]): An AWS_SESSION_TOKEN if using temporary + credentials. + """ + self.algorithm = 'AWS4-HMAC-SHA256' + self.expires = '900' + self.hashfunc = hashlib.sha256 + self.headers = [ + ('host', host) + ] + self.version = '2020_10_22' + + self.service = 'kafka-cluster' + self.action = '{}:Connect'.format(self.service) + + now = datetime.datetime.utcnow() + self.datestamp = now.strftime('%Y%m%d') + self.timestamp = now.strftime('%Y%m%dT%H%M%SZ') + + self.host = host + self.access_key = access_key + self.secret_key = secret_key + self.region = region + self.token = token + + @property + def _credential(self): + return '{0.access_key}/{0._scope}'.format(self) + + @property + def _scope(self): + return '{0.datestamp}/{0.region}/{0.service}/aws4_request'.format(self) + + @property + def _signed_headers(self): + """ + Returns (str): + An alphabetically sorted, semicolon-delimited list of lowercase + request header names. + """ + return ';'.join(sorted(k.lower() for k, _ in self.headers)) + + @property + def _canonical_headers(self): + """ + Returns (str): + A newline-delited list of header names and values. + Header names are lowercased. + """ + return '\n'.join(map(':'.join, self.headers)) + '\n' + + @property + def _canonical_request(self): + """ + Returns (str): + An AWS Signature Version 4 canonical request in the format: + \n + \n + \n + \n + \n + + """ + # The hashed_payload is always an empty string for MSK. + hashed_payload = self.hashfunc(b'').hexdigest() + return '\n'.join(( + 'GET', + '/', + self._canonical_querystring, + self._canonical_headers, + self._signed_headers, + hashed_payload, + )) + + @property + def _canonical_querystring(self): + """ + Returns (str): + A '&'-separated list of URI-encoded key/value pairs. + """ + params = [] + params.append(('Action', self.action)) + params.append(('X-Amz-Algorithm', self.algorithm)) + params.append(('X-Amz-Credential', self._credential)) + params.append(('X-Amz-Date', self.timestamp)) + params.append(('X-Amz-Expires', self.expires)) + if self.token: + params.append(('X-Amz-Security-Token', self.token)) + params.append(('X-Amz-SignedHeaders', self._signed_headers)) + + return '&'.join(self._uriencode(k) + '=' + self._uriencode(v) for k, v in params) + + @property + def _signing_key(self): + """ + Returns (bytes): + An AWS Signature V4 signing key generated from the secret_key, date, + region, service, and request type. + """ + key = self._hmac(('AWS4' + self.secret_key).encode('utf-8'), self.datestamp) + key = self._hmac(key, self.region) + key = self._hmac(key, self.service) + key = self._hmac(key, 'aws4_request') + return key + + @property + def _signing_str(self): + """ + Returns (str): + A string used to sign the AWS Signature V4 payload in the format: + \n + \n + \n + + """ + canonical_request_hash = self.hashfunc(self._canonical_request.encode('utf-8')).hexdigest() + return '\n'.join((self.algorithm, self.timestamp, self._scope, canonical_request_hash)) + + def _uriencode(self, msg): + """ + Arguments: + msg (str): A string to URI-encode. + + Returns (str): + The URI-encoded version of the provided msg, following the encoding + rules specified: https://github.com/aws/aws-msk-iam-auth#uriencode + """ + return urllib.parse.quote(msg, safe=self.UNRESERVED_CHARS) + + def _hmac(self, key, msg): + """ + Arguments: + key (bytes): A key to use for the HMAC digest. + msg (str): A value to include in the HMAC digest. + Returns (bytes): + An HMAC digest of the given key and msg. + """ + return hmac.new(key, msg.encode('utf-8'), digestmod=self.hashfunc).digest() + + def first_message(self): + """ + Returns (bytes): + An encoded JSON authentication payload that can be sent to the + broker. + """ + signature = hmac.new( + self._signing_key, + self._signing_str.encode('utf-8'), + digestmod=self.hashfunc, + ).hexdigest() + msg = { + 'version': self.version, + 'host': self.host, + 'user-agent': 'kafka-python', + 'action': self.action, + 'x-amz-algorithm': self.algorithm, + 'x-amz-credential': self._credential, + 'x-amz-date': self.timestamp, + 'x-amz-signedheaders': self._signed_headers, + 'x-amz-expires': self.expires, + 'x-amz-signature': signature, + } + if self.token: + msg['x-amz-security-token'] = self.token + + return json.dumps(msg, separators=(',', ':')).encode('utf-8') diff --git a/kafka/sasl/oauth.py b/kafka/sasl/oauth.py new file mode 100644 index 000000000..d4f643d84 --- /dev/null +++ b/kafka/sasl/oauth.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import + +from kafka.sasl.abc import SaslMechanism + + +class SaslMechanismOAuth(SaslMechanism): + + def __init__(self, **config): + assert 'sasl_oauth_token_provider' in config, 'sasl_oauth_token_provider required for OAUTHBEARER sasl' + self.token_provider = config['sasl_oauth_token_provider'] + assert callable(getattr(self.token_provider, 'token', None)), 'sasl_oauth_token_provider must implement method #token()' + self._is_done = False + self._is_authenticated = False + + def auth_bytes(self): + token = self.token_provider.token() + extensions = self._token_extensions() + return "n,,\x01auth=Bearer {}{}\x01\x01".format(token, extensions).encode('utf-8') + + def receive(self, auth_bytes): + self._is_done = True + self._is_authenticated = auth_bytes == b'' + + def is_done(self): + return self._is_done + + def is_authenticated(self): + return self._is_authenticated + + def _token_extensions(self): + """ + Return a string representation of the OPTIONAL key-value pairs that can be sent with an OAUTHBEARER + initial request. + """ + # Only run if the #extensions() method is implemented by the clients Token Provider class + # Builds up a string separated by \x01 via a dict of key value pairs + extensions = getattr(self.token_provider, 'extensions', lambda: [])() + msg = '\x01'.join(['{}={}'.format(k, v) for k, v in extensions.items()]) + return '\x01' + msg if msg else '' + + def auth_details(self): + if not self.is_authenticated: + raise RuntimeError('Not authenticated yet!') + return 'Authenticated via SASL / OAuth' diff --git a/kafka/sasl/plain.py b/kafka/sasl/plain.py new file mode 100644 index 000000000..81443f5fe --- /dev/null +++ b/kafka/sasl/plain.py @@ -0,0 +1,41 @@ +from __future__ import absolute_import + +import logging + +from kafka.sasl.abc import SaslMechanism + + +log = logging.getLogger(__name__) + + +class SaslMechanismPlain(SaslMechanism): + + def __init__(self, **config): + if config.get('security_protocol', '') == 'SASL_PLAINTEXT': + log.warning('Sending username and password in the clear') + assert 'sasl_plain_username' in config, 'sasl_plain_username required for PLAIN sasl' + assert 'sasl_plain_password' in config, 'sasl_plain_password required for PLAIN sasl' + + self.username = config['sasl_plain_username'] + self.password = config['sasl_plain_password'] + self._is_done = False + self._is_authenticated = False + + def auth_bytes(self): + # Send PLAIN credentials per RFC-4616 + return bytes('\0'.join([self.username, self.username, self.password]).encode('utf-8')) + + def receive(self, auth_bytes): + self._is_done = True + self._is_authenticated = auth_bytes == b'' + + def is_done(self): + return self._is_done + + def is_authenticated(self): + return self._is_authenticated + + def auth_details(self): + if not self.is_authenticated: + raise RuntimeError('Not authenticated yet!') + return 'Authenticated as %s via SASL / Plain' % self.username diff --git a/kafka/sasl/scram.py b/kafka/sasl/scram.py new file mode 100644 index 000000000..d8cd071a7 --- /dev/null +++ b/kafka/sasl/scram.py @@ -0,0 +1,133 @@ +from __future__ import absolute_import + +import base64 +import hashlib +import hmac +import logging +import uuid + + +from kafka.sasl.abc import SaslMechanism +from kafka.vendor import six + + +log = logging.getLogger(__name__) + + +if six.PY2: + def xor_bytes(left, right): + return bytearray(ord(lb) ^ ord(rb) for lb, rb in zip(left, right)) +else: + def xor_bytes(left, right): + return bytes(lb ^ rb for lb, rb in zip(left, right)) + + +class SaslMechanismScram(SaslMechanism): + def __init__(self, **config): + assert 'sasl_plain_username' in config, 'sasl_plain_username required for SCRAM sasl' + assert 'sasl_plain_password' in config, 'sasl_plain_password required for SCRAM sasl' + assert config.get('sasl_mechanism', '') in ScramClient.MECHANISMS, 'Unrecognized SCRAM mechanism' + if config.get('security_protocol', '') == 'SASL_PLAINTEXT': + log.warning('Exchanging credentials in the clear during Sasl Authentication') + + self.username = config['sasl_plain_username'] + self.mechanism = config['sasl_mechanism'] + self._scram_client = ScramClient( + config['sasl_plain_username'], + config['sasl_plain_password'], + config['sasl_mechanism'] + ) + self._state = 0 + + def auth_bytes(self): + if self._state == 0: + return self._scram_client.first_message() + elif self._state == 1: + return self._scram_client.final_message() + else: + raise ValueError('No auth_bytes for state: %s' % self._state) + + def receive(self, auth_bytes): + if self._state == 0: + self._scram_client.process_server_first_message(auth_bytes) + elif self._state == 1: + self._scram_client.process_server_final_message(auth_bytes) + else: + raise ValueError('Cannot receive bytes in state: %s' % self._state) + self._state += 1 + return self.is_done() + + def is_done(self): + return self._state == 2 + + def is_authenticated(self): + # receive raises if authentication fails...? + return self._state == 2 + + def auth_details(self): + if not self.is_authenticated: + raise RuntimeError('Not authenticated yet!') + return 'Authenticated as %s via SASL / %s' % (self.username, self.mechanism) + + +class ScramClient: + MECHANISMS = { + 'SCRAM-SHA-256': hashlib.sha256, + 'SCRAM-SHA-512': hashlib.sha512 + } + + def __init__(self, user, password, mechanism): + self.nonce = str(uuid.uuid4()).replace('-', '').encode('utf-8') + self.auth_message = b'' + self.salted_password = None + self.user = user.encode('utf-8') + self.password = password.encode('utf-8') + self.hashfunc = self.MECHANISMS[mechanism] + self.hashname = ''.join(mechanism.lower().split('-')[1:3]) + self.stored_key = None + self.client_key = None + self.client_signature = None + self.client_proof = None + self.server_key = None + self.server_signature = None + + def first_message(self): + client_first_bare = b'n=' + self.user + b',r=' + self.nonce + self.auth_message += client_first_bare + return b'n,,' + client_first_bare + + def process_server_first_message(self, server_first_message): + self.auth_message += b',' + server_first_message + params = dict(pair.split('=', 1) for pair in server_first_message.decode('utf-8').split(',')) + server_nonce = params['r'].encode('utf-8') + if not server_nonce.startswith(self.nonce): + raise ValueError("Server nonce, did not start with client nonce!") + self.nonce = server_nonce + self.auth_message += b',c=biws,r=' + self.nonce + + salt = base64.b64decode(params['s'].encode('utf-8')) + iterations = int(params['i']) + self.create_salted_password(salt, iterations) + + self.client_key = self.hmac(self.salted_password, b'Client Key') + self.stored_key = self.hashfunc(self.client_key).digest() + self.client_signature = self.hmac(self.stored_key, self.auth_message) + self.client_proof = xor_bytes(self.client_key, self.client_signature) + self.server_key = self.hmac(self.salted_password, b'Server Key') + self.server_signature = self.hmac(self.server_key, self.auth_message) + + def hmac(self, key, msg): + return hmac.new(key, msg, digestmod=self.hashfunc).digest() + + def create_salted_password(self, salt, iterations): + self.salted_password = hashlib.pbkdf2_hmac( + self.hashname, self.password, salt, iterations + ) + + def final_message(self): + return b'c=biws,r=' + self.nonce + b',p=' + base64.b64encode(self.client_proof) + + def process_server_final_message(self, server_final_message): + params = dict(pair.split('=', 1) for pair in server_final_message.decode('utf-8').split(',')) + if self.server_signature != base64.b64decode(params['v'].encode('utf-8')): + raise ValueError("Server sent wrong signature!") diff --git a/kafka/sasl/sspi.py b/kafka/sasl/sspi.py new file mode 100644 index 000000000..f4c95d037 --- /dev/null +++ b/kafka/sasl/sspi.py @@ -0,0 +1,111 @@ +from __future__ import absolute_import + +import logging + +# Windows-only +try: + import sspi + import pywintypes + import sspicon + import win32security +except ImportError: + sspi = None + +from kafka.sasl.abc import SaslMechanism + + +log = logging.getLogger(__name__) + + +class SaslMechanismSSPI(SaslMechanism): + # Establish security context and negotiate protection level + # For reference see RFC 4752, section 3 + + SASL_QOP_AUTH = 1 + SASL_QOP_AUTH_INT = 2 + SASL_QOP_AUTH_CONF = 4 + + def __init__(self, **config): + assert sspi is not None, 'No GSSAPI lib available (gssapi or sspi)' + if 'sasl_kerberos_name' not in config and 'sasl_kerberos_service_name' not in config: + raise ValueError('sasl_kerberos_service_name or sasl_kerberos_name required for GSSAPI sasl configuration') + self._is_done = False + self._is_authenticated = False + if config.get('sasl_kerberos_name', None) is not None: + self.auth_id = str(config['sasl_kerberos_name']) + else: + kerberos_domain_name = config.get('sasl_kerberos_domain_name', '') or config.get('host', '') + self.auth_id = config['sasl_kerberos_service_name'] + '/' + kerberos_domain_name + scheme = "Kerberos" # Do not try with Negotiate for SASL authentication. Tokens are different. + # https://docs.microsoft.com/en-us/windows/win32/secauthn/context-requirements + flags = ( + sspicon.ISC_REQ_MUTUAL_AUTH | # mutual authentication + sspicon.ISC_REQ_INTEGRITY | # check for integrity + sspicon.ISC_REQ_SEQUENCE_DETECT | # enable out-of-order messages + sspicon.ISC_REQ_CONFIDENTIALITY # request confidentiality + ) + self._client_ctx = sspi.ClientAuth(scheme, targetspn=self.auth_id, scflags=flags) + self._next_token = self._client_ctx.step(None) + + def auth_bytes(self): + # GSSAPI Auth does not have a final broker->client message + # so mark is_done after the final auth_bytes are provided + # in practice we'll still receive a response when using SaslAuthenticate + # but not when using the prior unframed approach. + if self._client_ctx.authenticated: + self._is_done = True + self._is_authenticated = True + return self._next_token or b'' + + def receive(self, auth_bytes): + log.debug("Received token from server (size %s)", len(auth_bytes)) + if not self._client_ctx.authenticated: + # calculate an output token from kafka token (or None on first iteration) + # https://docs.microsoft.com/en-us/windows/win32/api/sspi/nf-sspi-initializesecuritycontexta + # https://docs.microsoft.com/en-us/windows/win32/secauthn/initializesecuritycontext--kerberos + # authorize method will wrap for us our token in sspi structures + error, auth = self._client_ctx.authorize(auth_bytes) + if len(auth) > 0 and len(auth[0].Buffer): + log.debug("Got token from context") + # this buffer must be sent to the server whatever the result is + self._next_token = auth[0].Buffer + else: + log.debug("Got no token, exchange finished") + # seems to be the end of the loop + self._next_token = b'' + elif self._is_done: + # The final step of gssapi is send, so we do not expect any additional bytes + # however, allow an empty message to support SaslAuthenticate response + if auth_bytes != b'': + raise ValueError("Unexpected receive auth_bytes after sasl/gssapi completion") + else: + # Process the security layer negotiation token, sent by the server + # once the security context is established. + + # The following part is required by SASL, but not by classic Kerberos. + # See RFC 4752 + + # unwraps message containing supported protection levels and msg size + msg, _was_encrypted = self._client_ctx.unwrap(auth_bytes) + + # Kafka currently doesn't support integrity or confidentiality security layers, so we + # simply set QoP to 'auth' only (first octet). We reuse the max message size proposed + # by the server + client_flags = self.SASL_QOP_AUTH + server_flags = msg[0] + message_parts = [ + bytes(client_flags & server_flags), + msg[:1], + self.auth_id.encode('utf-8'), + ] + # add authorization identity to the response, and GSS-wrap + self._next_token = self._client_ctx.wrap(b''.join(message_parts), False) + + def is_done(self): + return self._is_done + + def is_authenticated(self): + return self._is_authenticated + + def auth_details(self): + return 'Authenticated as %s to %s via SASL / SSPI/GSSAPI \\o/' % (self._client_ctx.initiator_name, self._client_ctx.service_name) diff --git a/kafka/scram.py b/kafka/scram.py deleted file mode 100644 index 7f003750c..000000000 --- a/kafka/scram.py +++ /dev/null @@ -1,81 +0,0 @@ -from __future__ import absolute_import - -import base64 -import hashlib -import hmac -import uuid - -from kafka.vendor import six - - -if six.PY2: - def xor_bytes(left, right): - return bytearray(ord(lb) ^ ord(rb) for lb, rb in zip(left, right)) -else: - def xor_bytes(left, right): - return bytes(lb ^ rb for lb, rb in zip(left, right)) - - -class ScramClient: - MECHANISMS = { - 'SCRAM-SHA-256': hashlib.sha256, - 'SCRAM-SHA-512': hashlib.sha512 - } - - def __init__(self, user, password, mechanism): - self.nonce = str(uuid.uuid4()).replace('-', '') - self.auth_message = '' - self.salted_password = None - self.user = user - self.password = password.encode('utf-8') - self.hashfunc = self.MECHANISMS[mechanism] - self.hashname = ''.join(mechanism.lower().split('-')[1:3]) - self.stored_key = None - self.client_key = None - self.client_signature = None - self.client_proof = None - self.server_key = None - self.server_signature = None - - def first_message(self): - client_first_bare = 'n={},r={}'.format(self.user, self.nonce) - self.auth_message += client_first_bare - return 'n,,' + client_first_bare - - def process_server_first_message(self, server_first_message): - self.auth_message += ',' + server_first_message - params = dict(pair.split('=', 1) for pair in server_first_message.split(',')) - server_nonce = params['r'] - if not server_nonce.startswith(self.nonce): - raise ValueError("Server nonce, did not start with client nonce!") - self.nonce = server_nonce - self.auth_message += ',c=biws,r=' + self.nonce - - salt = base64.b64decode(params['s'].encode('utf-8')) - iterations = int(params['i']) - self.create_salted_password(salt, iterations) - - self.client_key = self.hmac(self.salted_password, b'Client Key') - self.stored_key = self.hashfunc(self.client_key).digest() - self.client_signature = self.hmac(self.stored_key, self.auth_message.encode('utf-8')) - self.client_proof = xor_bytes(self.client_key, self.client_signature) - self.server_key = self.hmac(self.salted_password, b'Server Key') - self.server_signature = self.hmac(self.server_key, self.auth_message.encode('utf-8')) - - def hmac(self, key, msg): - return hmac.new(key, msg, digestmod=self.hashfunc).digest() - - def create_salted_password(self, salt, iterations): - self.salted_password = hashlib.pbkdf2_hmac( - self.hashname, self.password, salt, iterations - ) - - def final_message(self): - return 'c=biws,r={},p={}'.format(self.nonce, base64.b64encode(self.client_proof).decode('utf-8')) - - def process_server_final_message(self, server_final_message): - params = dict(pair.split('=', 1) for pair in server_final_message.split(',')) - if self.server_signature != base64.b64decode(params['v'].encode('utf-8')): - raise ValueError("Server sent wrong signature!") - - diff --git a/kafka/structs.py b/kafka/structs.py index bcb023670..16ba0daac 100644 --- a/kafka/structs.py +++ b/kafka/structs.py @@ -42,7 +42,7 @@ this partition metadata. """ PartitionMetadata = namedtuple("PartitionMetadata", - ["topic", "partition", "leader", "replicas", "isr", "error"]) + ["topic", "partition", "leader", "leader_epoch", "replicas", "isr", "offline_replicas", "error"]) """The Kafka offset commit API @@ -55,10 +55,10 @@ Keyword Arguments: offset (int): The offset to be committed metadata (str): Non-null metadata + leader_epoch (int): The last known epoch from the leader / broker """ OffsetAndMetadata = namedtuple("OffsetAndMetadata", - # TODO add leaderEpoch: OffsetAndMetadata(offset, leaderEpoch, metadata) - ["offset", "metadata"]) + ["offset", "metadata", "leader_epoch"]) """An offset and timestamp tuple @@ -66,9 +66,10 @@ Keyword Arguments: offset (int): An offset timestamp (int): The timestamp associated to the offset + leader_epoch (int): The last known epoch from the leader / broker """ OffsetAndTimestamp = namedtuple("OffsetAndTimestamp", - ["offset", "timestamp"]) + ["offset", "timestamp", "leader_epoch"]) MemberInformation = namedtuple("MemberInformation", ["member_id", "client_id", "client_host", "member_metadata", "member_assignment"]) diff --git a/kafka/vendor/socketpair.py b/kafka/vendor/socketpair.py index b55e629ee..54d908767 100644 --- a/kafka/vendor/socketpair.py +++ b/kafka/vendor/socketpair.py @@ -53,6 +53,23 @@ def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): raise finally: lsock.close() + + # Authenticating avoids using a connection from something else + # able to connect to {host}:{port} instead of us. + # We expect only AF_INET and AF_INET6 families. + try: + if ( + ssock.getsockname() != csock.getpeername() + or csock.getsockname() != ssock.getpeername() + ): + raise ConnectionError("Unexpected peer connection") + except: + # getsockname() and getpeername() can fail + # if either socket isn't connected. + ssock.close() + csock.close() + raise + return (ssock, csock) socket.socketpair = socketpair diff --git a/kafka/version.py b/kafka/version.py index 06306bd1f..83d888e17 100644 --- a/kafka/version.py +++ b/kafka/version.py @@ -1 +1 @@ -__version__ = '2.0.3-dev' +__version__ = '2.1.0.dev' diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..ddd40a08e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["setuptools>=61.2"] +build-backend = "setuptools.build_meta" + +[project] +name = "kafka-python" +dynamic = ["version"] +authors = [{name = "Dana Powers", email = "dana.powers@gmail.com"}] +description = "Pure Python client for Apache Kafka" +keywords = ["apache kafka", "kafka"] +readme = "README.rst" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Software Development :: Libraries :: Python Modules", +] +urls = {Homepage = "https://github.com/dpkp/kafka-python"} + +[project.optional-dependencies] +crc32c = ["crc32c"] +lz4 = ["lz4"] +snappy = ["python-snappy"] +zstd = ["zstandard"] +testing = ["pytest", "mock; python_version < '3.3'", "pytest-mock"] + +[tool.setuptools] +include-package-data = false +license-files = [] # workaround for https://github.com/pypa/setuptools/issues/4759 + +[tool.setuptools.packages.find] +exclude = ["test"] +namespaces = false + +[tool.distutils.bdist_wheel] +universal = 1 + +[tool.setuptools.dynamic] +version = {attr = "kafka.__version__"} diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..f54588733 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +log_format = %(created)f %(filename)-23s %(threadName)s %(message)s diff --git a/requirements-dev.txt b/requirements-dev.txt index 1fa933da2..6cfb6d83b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,7 @@ crc32c docker-py flake8 lz4 -mock +mock; python_version < '3.3' py pylint pytest @@ -13,5 +13,5 @@ pytest-pylint python-snappy Sphinx sphinx-rtd-theme -tox xxhash +zstandard diff --git a/servers/0.11.0.0/resources/kafka.properties b/servers/0.11.0.0/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/0.11.0.0/resources/kafka.properties +++ b/servers/0.11.0.0/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/0.11.0.1/resources/kafka.properties b/servers/0.11.0.1/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/0.11.0.1/resources/kafka.properties +++ b/servers/0.11.0.1/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/0.11.0.2/resources/kafka.properties b/servers/0.11.0.2/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/0.11.0.2/resources/kafka.properties +++ b/servers/0.11.0.2/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/0.11.0.3/resources/kafka.properties b/servers/0.11.0.3/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/0.11.0.3/resources/kafka.properties +++ b/servers/0.11.0.3/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/1.0.0/resources/kafka.properties b/servers/1.0.0/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/1.0.0/resources/kafka.properties +++ b/servers/1.0.0/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/1.0.1/resources/kafka.properties b/servers/1.0.1/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/1.0.1/resources/kafka.properties +++ b/servers/1.0.1/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/1.0.2/resources/kafka.properties b/servers/1.0.2/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/1.0.2/resources/kafka.properties +++ b/servers/1.0.2/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/1.1.0/resources/kafka.properties b/servers/1.1.0/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/1.1.0/resources/kafka.properties +++ b/servers/1.1.0/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/1.1.1/resources/kafka.properties b/servers/1.1.1/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/1.1.1/resources/kafka.properties +++ b/servers/1.1.1/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/2.0.0/resources/kafka.properties b/servers/2.0.0/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/2.0.0/resources/kafka.properties +++ b/servers/2.0.0/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/2.0.1/resources/kafka.properties b/servers/2.0.1/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/2.0.1/resources/kafka.properties +++ b/servers/2.0.1/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/2.1.0/resources/kafka.properties b/servers/2.1.0/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/2.1.0/resources/kafka.properties +++ b/servers/2.1.0/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/2.1.1/resources/kafka.properties b/servers/2.1.1/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/2.1.1/resources/kafka.properties +++ b/servers/2.1.1/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/2.2.1/resources/kafka.properties b/servers/2.2.1/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/2.2.1/resources/kafka.properties +++ b/servers/2.2.1/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/2.3.0/resources/kafka.properties b/servers/2.3.0/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/2.3.0/resources/kafka.properties +++ b/servers/2.3.0/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/2.4.0/resources/kafka.properties b/servers/2.4.0/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/2.4.0/resources/kafka.properties +++ b/servers/2.4.0/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/2.5.0/resources/kafka.properties b/servers/2.5.0/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/2.5.0/resources/kafka.properties +++ b/servers/2.5.0/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/2.6.0/resources/kafka.properties b/servers/2.6.0/resources/kafka.properties index 5775cfdc4..219023551 100644 --- a/servers/2.6.0/resources/kafka.properties +++ b/servers/2.6.0/resources/kafka.properties @@ -4,14 +4,15 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + # see kafka.server.KafkaConfig for additional details and defaults ############################# Server Basics ############################# @@ -21,6 +22,12 @@ broker.id={broker_id} ############################# Socket Server Settings ############################# +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 listeners={transport}://{host}:{port} security.inter.broker.protocol={transport} @@ -38,22 +45,18 @@ allow.everyone.if.no.acl.found=true # The port the socket server listens on #port=9092 -# Hostname the broker will bind to. If not set, the server will bind to all interfaces -#host.name=localhost - -# Hostname the broker will advertise to producers and consumers. If not set, it uses the -# value for "host.name" if configured. Otherwise, it will use the value returned from -# java.net.InetAddress.getCanonicalHostName(). -#advertised.host.name= +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 -# The port to publish to ZooKeeper for clients to use. If this is not set, -# it will publish the same port that the broker binds to. -#advertised.port= +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -# The number of threads handling network requests +# The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 - -# The number of threads doing disk I/O + +# The number of threads that the server uses for processing requests, which may include disk I/O num.io.threads=8 # The send buffer (SO_SNDBUF) used by the socket server @@ -68,7 +71,7 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# A comma seperated list of directories under which to store log files +# A comma separated list of directories under which to store log files log.dirs={tmp_dir}/data # The default number of log partitions per topic. More partitions allow greater @@ -81,14 +84,25 @@ default.replication.factor={replicas} replica.lag.time.max.ms=1000 replica.socket.timeout.ms=1000 +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + ############################# Log Flush Policy ############################# # Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. +# the OS cache lazily. The following configurations control the flush of data to disk. # There are a few important trade-offs here: # 1. Durability: Unflushed data may be lost if you are not using replication. # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -105,17 +119,17 @@ replica.socket.timeout.ms=1000 # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens # from the end of the log. -# The minimum age of a log file to be eligible for deletion +# The minimum age of a log file to be eligible for deletion due to age log.retention.hours=168 -# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining -# segments don't drop below log.retention.bytes. +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. #log.retention.bytes=1073741824 # The maximum size of a log segment file. When this size is reached a new log segment will be created. log.segment.bytes=1073741824 -# The interval at which log segments are checked to see if they can be deleted according +# The interval at which log segments are checked to see if they can be deleted according # to the retention policies log.retention.check.interval.ms=300000 @@ -145,3 +159,13 @@ zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} zookeeper.connection.timeout.ms=30000 # We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/resources/default/kafka.properties b/servers/resources/default/kafka.properties new file mode 100644 index 000000000..71b20f53e --- /dev/null +++ b/servers/resources/default/kafka.properties @@ -0,0 +1,171 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# see kafka.server.KafkaConfig for additional details and defaults + +############################# Server Basics ############################# + +# The id of the broker. This must be set to a unique integer for each broker. +broker.id={broker_id} + +############################# Socket Server Settings ############################# + +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 +listeners={transport}://{host}:{port} +security.inter.broker.protocol={transport} + +{sasl_config} + +ssl.keystore.location={ssl_dir}/kafka.server.keystore.jks +ssl.keystore.password=foobar +ssl.key.password=foobar +ssl.truststore.location={ssl_dir}/kafka.server.truststore.jks +ssl.truststore.password=foobar + +authorizer.class.name=kafka.security.authorizer.AclAuthorizer +allow.everyone.if.no.acl.found=true + +# The port the socket server listens on +#port=9092 + +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +#advertised.listeners=PLAINTEXT://your.host.name:9092 + +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + +# The number of threads that the server uses for receiving requests from the network and sending responses to the network +num.network.threads=3 + +# The number of threads that the server uses for processing requests, which may include disk I/O +num.io.threads=8 + +# The send buffer (SO_SNDBUF) used by the socket server +socket.send.buffer.bytes=102400 + +# The receive buffer (SO_RCVBUF) used by the socket server +socket.receive.buffer.bytes=102400 + +# The maximum size of a request that the socket server will accept (protection against OOM) +socket.request.max.bytes=104857600 + + +############################# Log Basics ############################# + +# A comma separated list of directories under which to store log files +log.dirs={tmp_dir}/data + +# The default number of log partitions per topic. More partitions allow greater +# parallelism for consumption, but this will also result in more files across +# the brokers. +num.partitions={partitions} +default.replication.factor={replicas} + +## Short Replica Lag -- Drops failed brokers out of ISR +replica.lag.time.max.ms=1000 +replica.socket.timeout.ms=1000 + +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. +offsets.topic.replication.factor=1 +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + +############################# Log Flush Policy ############################# + +# Messages are immediately written to the filesystem but by default we only fsync() to sync +# the OS cache lazily. The following configurations control the flush of data to disk. +# There are a few important trade-offs here: +# 1. Durability: Unflushed data may be lost if you are not using replication. +# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. +# The settings below allow one to configure the flush policy to flush data after a period of time or +# every N messages (or both). This can be done globally and overridden on a per-topic basis. + +# The number of messages to accept before forcing a flush of data to disk +#log.flush.interval.messages=10000 + +# The maximum amount of time a message can sit in a log before we force a flush +#log.flush.interval.ms=1000 + +############################# Log Retention Policy ############################# + +# The following configurations control the disposal of log segments. The policy can +# be set to delete segments after a period of time, or after a given size has accumulated. +# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens +# from the end of the log. + +# The minimum age of a log file to be eligible for deletion due to age +log.retention.hours=168 + +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. +#log.retention.bytes=1073741824 + +# The maximum size of a log segment file. When this size is reached a new log segment will be created. +log.segment.bytes=1073741824 + +# The interval at which log segments are checked to see if they can be deleted according +# to the retention policies +log.retention.check.interval.ms=300000 + +# By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires. +# If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction. +log.cleaner.enable=false + +# tune down offset topics to reduce setup time in tests +offsets.commit.timeout.ms=500 +offsets.topic.num.partitions=2 +offsets.topic.replication.factor=1 + +# Allow shorter session timeouts for tests +group.min.session.timeout.ms=1000 + + +############################# Zookeeper ############################# + +# Zookeeper connection string (see zookeeper docs for details). +# This is a comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". +# You can also append an optional chroot string to the urls to specify the +# root directory for all kafka znodes. +zookeeper.connect={zk_host}:{zk_port}/{zk_chroot} + +# Timeout in ms for connecting to zookeeper +zookeeper.connection.timeout.ms=30000 +# We want to expire kafka broker sessions quickly when brokers die b/c we restart them quickly +zookeeper.session.timeout.ms=500 + + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/servers/resources/default/kafka_server_jaas.conf b/servers/resources/default/kafka_server_jaas.conf new file mode 100644 index 000000000..18efe4369 --- /dev/null +++ b/servers/resources/default/kafka_server_jaas.conf @@ -0,0 +1,4 @@ +KafkaServer {{ + {jaas_config} +}}; +Client {{}}; \ No newline at end of file diff --git a/servers/resources/default/log4j.properties b/servers/resources/default/log4j.properties new file mode 100644 index 000000000..b0b76aa79 --- /dev/null +++ b/servers/resources/default/log4j.properties @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=INFO, stdout, logfile + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.logfile=org.apache.log4j.FileAppender +log4j.appender.logfile.File=${kafka.logs.dir}/server.log +log4j.appender.logfile.layout=org.apache.log4j.PatternLayout +log4j.appender.logfile.layout.ConversionPattern=[%d] %p %m (%c)%n diff --git a/servers/resources/default/sasl_command.conf b/servers/resources/default/sasl_command.conf new file mode 100644 index 000000000..f4ae7bafa --- /dev/null +++ b/servers/resources/default/sasl_command.conf @@ -0,0 +1,3 @@ +security.protocol={transport} +sasl.mechanism={sasl_mechanism} +sasl.jaas.config={jaas_config} diff --git a/servers/resources/default/zookeeper.properties b/servers/resources/default/zookeeper.properties new file mode 100644 index 000000000..e3fd09742 --- /dev/null +++ b/servers/resources/default/zookeeper.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# the directory where the snapshot is stored. +dataDir={tmp_dir} +# the port at which the clients will connect +clientPort={port} +clientPortAddress={host} +# disable the per-ip limit on the number of connections since this is a non-production config +maxClientCnxns=0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 5c6311daf..000000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[bdist_wheel] -universal=1 - -[metadata] -license_file = LICENSE diff --git a/setup.py b/setup.py index 77043da04..87b428a4e 100644 --- a/setup.py +++ b/setup.py @@ -1,78 +1,4 @@ -import os -import sys +# See pyproject.toml for project / build configuration +from setuptools import setup -from setuptools import setup, Command, find_packages - -# Pull version from source without importing -# since we can't import something we haven't built yet :) -exec(open('kafka/version.py').read()) - - -class Tox(Command): - - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - @classmethod - def run(cls): - import tox - sys.exit(tox.cmdline([])) - - -test_require = ['tox', 'mock'] - -here = os.path.abspath(os.path.dirname(__file__)) - -with open(os.path.join(here, 'README.rst')) as f: - README = f.read() - -setup( - name="kafka-python", - version=__version__, - - tests_require=test_require, - extras_require={ - "crc32c": ["crc32c"], - "lz4": ["lz4"], - "snappy": ["python-snappy"], - "zstd": ["zstandard"], - }, - cmdclass={"test": Tox}, - packages=find_packages(exclude=['test']), - author="Dana Powers", - author_email="dana.powers@gmail.com", - url="https://github.com/dpkp/kafka-python", - license="Apache License 2.0", - description="Pure Python client for Apache Kafka", - long_description=README, - keywords=[ - "apache kafka", - "kafka", - ], - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: Software Development :: Libraries :: Python Modules", - ] -) +setup() diff --git a/test/conftest.py b/test/conftest.py index 3fa0262fd..d54a91243 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -137,7 +137,9 @@ def conn(mocker): MetadataResponse[0]( [(0, 'foo', 12), (1, 'bar', 34)], # brokers [])) # topics + conn.connection_delay.return_value = 0 conn.blacked_out.return_value = False + conn.next_ifr_request_timeout_ms.return_value = float('inf') def _set_conn_state(state): conn.state = state return state diff --git a/test/fixtures.py b/test/fixtures.py index d9c072b86..f8e2aa746 100644 --- a/test/fixtures.py +++ b/test/fixtures.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, division import atexit import logging @@ -71,6 +71,8 @@ class Fixture(object): def __init__(self): self.child = None + if not os.path.isdir(self.kafka_root): + raise FileNotFoundError(self.kafka_root) @classmethod def download_official_distribution(cls, @@ -111,7 +113,16 @@ def download_official_distribution(cls, @classmethod def test_resource(cls, filename): - return os.path.join(cls.project_root, "servers", cls.kafka_version, "resources", filename) + path = os.path.join(cls.project_root, "servers", cls.kafka_version, "resources", filename) + if os.path.isfile(path): + return path + return os.path.join(cls.project_root, "servers", "resources", "default", filename) + + @classmethod + def run_script(cls, script, *args): + result = [os.path.join(cls.kafka_root, 'bin', script)] + result.extend([str(arg) for arg in args]) + return result @classmethod def kafka_run_class_args(cls, *args): @@ -183,7 +194,8 @@ def kafka_run_class_env(self): return env def out(self, message): - log.info("*** Zookeeper [%s:%s]: %s", self.host, self.port or '(auto)', message) + if len(log.handlers) > 0: + log.info("*** Zookeeper [%s:%s]: %s", self.host, self.port or '(auto)', message) def open(self): if self.tmp_dir is None: @@ -198,6 +210,7 @@ def open(self): # Configure Zookeeper child process template = self.test_resource("zookeeper.properties") properties = self.tmp_dir.join("zookeeper.properties") + # Consider replacing w/ run_script('zookeper-server-start.sh', ...) args = self.kafka_run_class_args("org.apache.zookeeper.server.quorum.QuorumPeerMain", properties.strpath) env = self.kafka_run_class_env() @@ -330,13 +343,13 @@ def _jaas_config(self): elif self.sasl_mechanism == 'PLAIN': jaas_config = ( - 'org.apache.kafka.common.security.plain.PlainLoginModule required\n' - ' username="{user}" password="{password}" user_{user}="{password}";\n' + 'org.apache.kafka.common.security.plain.PlainLoginModule required' + ' username="{user}" password="{password}" user_{user}="{password}";\n' ) elif self.sasl_mechanism in ("SCRAM-SHA-256", "SCRAM-SHA-512"): jaas_config = ( - 'org.apache.kafka.common.security.scram.ScramLoginModule required\n' - ' username="{user}" password="{password}";\n' + 'org.apache.kafka.common.security.scram.ScramLoginModule required' + ' username="{user}" password="{password}";\n' ) else: raise ValueError("SASL mechanism {} currently not supported".format(self.sasl_mechanism)) @@ -344,18 +357,16 @@ def _jaas_config(self): def _add_scram_user(self): self.out("Adding SCRAM credentials for user {} to zookeeper.".format(self.broker_user)) - args = self.kafka_run_class_args( - "kafka.admin.ConfigCommand", - "--zookeeper", - "%s:%d/%s" % (self.zookeeper.host, - self.zookeeper.port, - self.zk_chroot), - "--alter", - "--entity-type", "users", - "--entity-name", self.broker_user, - "--add-config", - "{}=[password={}]".format(self.sasl_mechanism, self.broker_password), - ) + args = self.run_script('kafka-configs.sh', + '--zookeeper', + '%s:%d/%s' % (self.zookeeper.host, + self.zookeeper.port, + self.zk_chroot), + '--alter', + '--entity-type', 'users', + '--entity-name', self.broker_user, + '--add-config', + '{}=[password={}]'.format(self.sasl_mechanism, self.broker_password)) env = self.kafka_run_class_env() proc = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -381,17 +392,17 @@ def kafka_run_class_env(self): return env def out(self, message): - log.info("*** Kafka [%s:%s]: %s", self.host, self.port or '(auto)', message) + if len(log.handlers) > 0: + log.info("*** Kafka [%s:%s]: %s", self.host, self.port or '(auto)', message) def _create_zk_chroot(self): self.out("Creating Zookeeper chroot node...") - args = self.kafka_run_class_args("org.apache.zookeeper.ZooKeeperMain", - "-server", - "%s:%d" % (self.zookeeper.host, - self.zookeeper.port), - "create", - "/%s" % (self.zk_chroot,), - "kafka-python") + args = self.run_script('zookeeper-shell.sh', + '%s:%d' % (self.zookeeper.host, + self.zookeeper.port), + 'create', + '/%s' % (self.zk_chroot,), + 'kafka-python') env = self.kafka_run_class_env() proc = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -411,6 +422,7 @@ def start(self): properties_template = self.test_resource("kafka.properties") jaas_conf_template = self.test_resource("kafka_server_jaas.conf") + # Consider replacing w/ run_script('kafka-server-start.sh', ...) args = self.kafka_run_class_args("kafka.Kafka", properties.strpath) env = self.kafka_run_class_env() if self.sasl_enabled: @@ -536,7 +548,7 @@ def _failure(error): break self._client.poll(timeout_ms=100) else: - raise RuntimeError('Could not connect to broker with node id %d' % (node_id,)) + raise RuntimeError('Could not connect to broker with node id %s' % (node_id,)) try: future = self._client.send(node_id, request) @@ -573,7 +585,15 @@ def _create_topic(self, topic_name, num_partitions=None, replication_factor=None self._create_topic_via_cli(topic_name, num_partitions, replication_factor) def _create_topic_via_metadata(self, topic_name, timeout_ms=10000): - self._send_request(MetadataRequest[0]([topic_name]), timeout_ms) + timeout_at = time.time() + timeout_ms / 1000 + while time.time() < timeout_at: + response = self._send_request(MetadataRequest[0]([topic_name]), timeout_ms) + if response.topics[0][0] == 0: + return + log.warning("Unable to create topic via MetadataRequest: err %d", response.topics[0][0]) + time.sleep(1) + else: + raise RuntimeError('Unable to create topic via MetadataRequest') def _create_topic_via_admin_api(self, topic_name, num_partitions, replication_factor, timeout_ms=10000): request = CreateTopicsRequest[0]([(topic_name, num_partitions, @@ -585,17 +605,15 @@ def _create_topic_via_admin_api(self, topic_name, num_partitions, replication_fa raise errors.for_code(error_code) def _create_topic_via_cli(self, topic_name, num_partitions, replication_factor): - args = self.kafka_run_class_args('kafka.admin.TopicCommand', - '--zookeeper', '%s:%s/%s' % (self.zookeeper.host, - self.zookeeper.port, - self.zk_chroot), - '--create', - '--topic', topic_name, - '--partitions', self.partitions \ - if num_partitions is None else num_partitions, - '--replication-factor', self.replicas \ - if replication_factor is None \ - else replication_factor) + args = self.run_script('kafka-topics.sh', + '--create', + '--topic', topic_name, + '--partitions', self.partitions \ + if num_partitions is None else num_partitions, + '--replication-factor', self.replicas \ + if replication_factor is None \ + else replication_factor, + *self._cli_connect_args()) if env_kafka_version() >= (0, 10): args.append('--if-not-exists') env = self.kafka_run_class_env() @@ -608,16 +626,23 @@ def _create_topic_via_cli(self, topic_name, num_partitions, replication_factor): self.out(stderr) raise RuntimeError("Failed to create topic %s" % (topic_name,)) + def _cli_connect_args(self): + if env_kafka_version() < (3, 0, 0): + return ['--zookeeper', '%s:%s/%s' % (self.zookeeper.host, self.zookeeper.port, self.zk_chroot)] + else: + args = ['--bootstrap-server', '%s:%s' % (self.host, self.port)] + if self.sasl_enabled: + command_conf = self.tmp_dir.join("sasl_command.conf") + self.render_template(self.test_resource("sasl_command.conf"), command_conf, vars(self)) + args.append('--command-config') + args.append(command_conf.strpath) + return args + def get_topic_names(self): - args = self.kafka_run_class_args('kafka.admin.TopicCommand', - '--zookeeper', '%s:%s/%s' % (self.zookeeper.host, - self.zookeeper.port, - self.zk_chroot), - '--list' - ) + cmd = self.run_script('kafka-topics.sh', '--list', *self._cli_connect_args()) env = self.kafka_run_class_env() env.pop('KAFKA_LOG4J_OPTS') - proc = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate() if proc.returncode != 0: self.out("Failed to list topics!") @@ -671,3 +696,51 @@ def get_producers(self, cnt, **params): params = self._enrich_client_params(params, client_id='producer') for client in self._create_many_clients(cnt, KafkaProducer, **params): yield client + + +def get_api_versions(): + logging.basicConfig(level=logging.ERROR) + zk = ZookeeperFixture.instance() + k = KafkaFixture.instance(0, zk) + + from kafka import KafkaClient + client = KafkaClient(bootstrap_servers='localhost:{}'.format(k.port)) + client.check_version() + + from pprint import pprint + + pprint(client.get_api_versions()) + + client.close() + k.close() + zk.close() + + +def run_brokers(): + logging.basicConfig(level=logging.ERROR) + zk = ZookeeperFixture.instance() + k = KafkaFixture.instance(0, zk) + + print("Kafka", k.kafka_version, "running on port:", k.port) + try: + while True: + time.sleep(5) + except KeyboardInterrupt: + print("Bye!") + k.close() + zk.close() + + +if __name__ == '__main__': + import sys + if len(sys.argv) < 2: + print("Commands: get_api_versions") + exit(0) + cmd = sys.argv[1] + if cmd == 'get_api_versions': + get_api_versions() + elif cmd == 'kafka': + run_brokers() + else: + print("Unknown cmd: %s", cmd) + exit(1) diff --git a/test/record/test_default_records.py b/test/record/test_default_records.py index c3a7b02c8..e1c840fa6 100644 --- a/test/record/test_default_records.py +++ b/test/record/test_default_records.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals import pytest -from mock import patch +try: + from unittest.mock import patch +except ImportError: + from mock import patch import kafka.codec from kafka.record.default_records import ( DefaultRecordBatch, DefaultRecordBatchBuilder diff --git a/test/record/test_legacy_records.py b/test/record/test_legacy_records.py index 43970f7c9..b15b53704 100644 --- a/test/record/test_legacy_records.py +++ b/test/record/test_legacy_records.py @@ -1,6 +1,9 @@ from __future__ import unicode_literals import pytest -from mock import patch +try: + from unittest.mock import patch +except ImportError: + from mock import patch from kafka.record.legacy_records import ( LegacyRecordBatch, LegacyRecordBatchBuilder ) diff --git a/test/record/test_records.py b/test/record/test_records.py index 9f72234ae..cab95922d 100644 --- a/test/record/test_records.py +++ b/test/record/test_records.py @@ -60,6 +60,15 @@ b'\x00\xff\xff\xff\xff\x00\x00\x00\x03123' ] +# Single record control batch (abort) +control_batch_data_v2 = [ + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00R\x00\x00\x00\x00' + b'\x02e\x97\xff\xd0\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x98\x96\x7f\x00\x00\x00\x00\x00\x98\x96' + b'\x7f\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + b'\x00\x00\x00\x01@\x00\x00\x00\x08\x00\x00\x00\x00,opaque-control-message\x00' +] + def test_memory_records_v2(): data_bytes = b"".join(record_batch_data_v2) + b"\x00" * 4 @@ -198,7 +207,7 @@ def test_memory_records_builder(magic, compression_type): # Size should remain the same after closing. No trailing bytes builder.close() assert builder.compression_rate() > 0 - expected_size = size_before_close * builder.compression_rate() + expected_size = int(size_before_close * builder.compression_rate()) assert builder.is_full() assert builder.size_in_bytes() == expected_size buffer = builder.buffer() @@ -230,3 +239,18 @@ def test_memory_records_builder_full(magic, compression_type): key=None, timestamp=None, value=b"M") assert metadata is None assert builder.next_offset() == 1 + + +def test_control_record_v2(): + data_bytes = b"".join(control_batch_data_v2) + records = MemoryRecords(data_bytes) + + assert records.has_next() is True + batch = records.next_batch() + assert batch.is_control_batch is True + recs = list(batch) + assert len(recs) == 1 + assert recs[0].version == 0 + assert recs[0].type == 0 + assert recs[0].abort is True + assert recs[0].commit is False diff --git a/test/sasl/test_msk.py b/test/sasl/test_msk.py new file mode 100644 index 000000000..297ca84ce --- /dev/null +++ b/test/sasl/test_msk.py @@ -0,0 +1,67 @@ +import datetime +import json + +from kafka.sasl.msk import AwsMskIamClient + +try: + from unittest import mock +except ImportError: + import mock + + +def client_factory(token=None): + now = datetime.datetime.utcfromtimestamp(1629321911) + with mock.patch('kafka.sasl.msk.datetime') as mock_dt: + mock_dt.datetime.utcnow = mock.Mock(return_value=now) + return AwsMskIamClient( + host='localhost', + access_key='XXXXXXXXXXXXXXXXXXXX', + secret_key='XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', + region='us-east-1', + token=token, + ) + + +def test_aws_msk_iam_client_permanent_credentials(): + client = client_factory(token=None) + msg = client.first_message() + assert msg + assert isinstance(msg, bytes) + actual = json.loads(msg) + + expected = { + 'version': '2020_10_22', + 'host': 'localhost', + 'user-agent': 'kafka-python', + 'action': 'kafka-cluster:Connect', + 'x-amz-algorithm': 'AWS4-HMAC-SHA256', + 'x-amz-credential': 'XXXXXXXXXXXXXXXXXXXX/20210818/us-east-1/kafka-cluster/aws4_request', + 'x-amz-date': '20210818T212511Z', + 'x-amz-signedheaders': 'host', + 'x-amz-expires': '900', + 'x-amz-signature': '0fa42ae3d5693777942a7a4028b564f0b372bafa2f71c1a19ad60680e6cb994b', + } + assert actual == expected + + +def test_aws_msk_iam_client_temporary_credentials(): + client = client_factory(token='XXXXX') + msg = client.first_message() + assert msg + assert isinstance(msg, bytes) + actual = json.loads(msg) + + expected = { + 'version': '2020_10_22', + 'host': 'localhost', + 'user-agent': 'kafka-python', + 'action': 'kafka-cluster:Connect', + 'x-amz-algorithm': 'AWS4-HMAC-SHA256', + 'x-amz-credential': 'XXXXXXXXXXXXXXXXXXXX/20210818/us-east-1/kafka-cluster/aws4_request', + 'x-amz-date': '20210818T212511Z', + 'x-amz-signedheaders': 'host', + 'x-amz-expires': '900', + 'x-amz-signature': 'b0619c50b7ecb4a7f6f92bd5f733770df5710e97b25146f97015c0b1db783b05', + 'x-amz-security-token': 'XXXXX', + } + assert actual == expected diff --git a/test/service.py b/test/service.py index 045d780e7..e4e89f8fe 100644 --- a/test/service.py +++ b/test/service.py @@ -59,7 +59,7 @@ def _spawn(self): self.args, preexec_fn=os.setsid, # to avoid propagating signals env=self.env, - bufsize=1, + bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.alive = self.child.poll() is None @@ -113,7 +113,8 @@ def wait_for(self, pattern, timeout=30): start = time.time() while True: if not self.is_alive(): - raise RuntimeError("Child thread died already.") + log.error("Child thread died already.") + return False elapsed = time.time() - start if elapsed >= timeout: diff --git a/test/test_admin.py b/test/test_admin.py index 279f85abf..cdb74242e 100644 --- a/test/test_admin.py +++ b/test/test_admin.py @@ -6,7 +6,7 @@ def test_config_resource(): with pytest.raises(KeyError): - bad_resource = kafka.admin.ConfigResource('something', 'foo') + _bad_resource = kafka.admin.ConfigResource('something', 'foo') good_resource = kafka.admin.ConfigResource('broker', 'bar') assert good_resource.resource_type == kafka.admin.ConfigResourceType.BROKER assert good_resource.name == 'bar' @@ -59,11 +59,11 @@ def test_acl_resource(): def test_new_topic(): with pytest.raises(IllegalArgumentError): - bad_topic = kafka.admin.NewTopic('foo', -1, -1) + _bad_topic = kafka.admin.NewTopic('foo', -1, -1) with pytest.raises(IllegalArgumentError): - bad_topic = kafka.admin.NewTopic('foo', 1, -1) + _bad_topic = kafka.admin.NewTopic('foo', 1, -1) with pytest.raises(IllegalArgumentError): - bad_topic = kafka.admin.NewTopic('foo', 1, 1, {1: [1, 1, 1]}) + _bad_topic = kafka.admin.NewTopic('foo', 1, 1, {1: [1, 1, 1]}) good_topic = kafka.admin.NewTopic('foo', 1, 2) assert good_topic.name == 'foo' assert good_topic.num_partitions == 1 diff --git a/test/test_admin_integration.py b/test/test_admin_integration.py index 06c40a223..2f6b76598 100644 --- a/test/test_admin_integration.py +++ b/test/test_admin_integration.py @@ -140,7 +140,7 @@ def test_describe_configs_invalid_broker_id_raises(kafka_admin_client): broker_id = "str" with pytest.raises(ValueError): - configs = kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.BROKER, broker_id)]) + kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.BROKER, broker_id)]) @pytest.mark.skipif(env_kafka_version() < (0, 11), reason='Describe consumer group requires broker >=0.11') @@ -148,7 +148,7 @@ def test_describe_consumer_group_does_not_exist(kafka_admin_client): """Tests that the describe consumer group call fails if the group coordinator is not available """ with pytest.raises(GroupCoordinatorNotAvailableError): - group_description = kafka_admin_client.describe_consumer_groups(['test']) + kafka_admin_client.describe_consumer_groups(['test']) @pytest.mark.skipif(env_kafka_version() < (0, 11), reason='Describe consumer group requires broker >=0.11') @@ -168,7 +168,7 @@ def consumer_thread(i, group_id): stop[i] = Event() consumers[i] = kafka_consumer_factory(group_id=group_id) while not stop[i].is_set(): - consumers[i].poll(20) + consumers[i].poll(timeout_ms=200) consumers[i].close() consumers[i] = None stop[i] = None @@ -183,6 +183,7 @@ def consumer_thread(i, group_id): try: timeout = time() + 35 while True: + info('Checking consumers...') for c in range(num_consumers): # Verify all consumers have been created @@ -212,9 +213,9 @@ def consumer_thread(i, group_id): if not rejoining and is_same_generation: break - else: - sleep(1) assert time() < timeout, "timeout waiting for assignments" + info('sleeping...') + sleep(1) info('Group stabilized; verifying assignment') output = kafka_admin_client.describe_consumer_groups(group_id_list) @@ -236,6 +237,8 @@ def consumer_thread(i, group_id): for c in range(num_consumers): info('Stopping consumer %s', c) stop[c].set() + for c in range(num_consumers): + info('Waiting for consumer thread %s', c) threads[c].join() threads[c] = None diff --git a/test/test_client_async.py b/test/test_client_async.py index 66b227aa9..015f39365 100644 --- a/test/test_client_async.py +++ b/test/test_client_async.py @@ -43,7 +43,7 @@ def test_bootstrap(mocker, conn): kwargs.pop('state_change_callback') kwargs.pop('node_id') assert kwargs == cli.config - conn.send.assert_called_once_with(MetadataRequest[0]([]), blocking=False) + conn.send.assert_called_once_with(MetadataRequest[0]([]), blocking=False, request_timeout_ms=None) assert cli._bootstrap_fails == 0 assert cli.cluster.brokers() == set([BrokerMetadata(0, 'foo', 12, None), BrokerMetadata(1, 'bar', 34, None)]) @@ -58,7 +58,7 @@ def test_can_connect(cli, conn): assert cli._can_connect(0) # Node is connected, can't reconnect - assert cli._maybe_connect(0) is True + assert cli._init_connect(0) is True assert not cli._can_connect(0) # Node is disconnected, can connect @@ -70,20 +70,15 @@ def test_can_connect(cli, conn): assert not cli._can_connect(0) -def test_maybe_connect(cli, conn): - try: - # Node not in metadata, raises AssertionError - cli._maybe_connect(2) - except AssertionError: - pass - else: - assert False, 'Exception not raised' +def test_init_connect(cli, conn): + # Node not in metadata, return False + assert not cli._init_connect(2) # New node_id creates a conn object assert 0 not in cli._conns conn.state = ConnectionStates.DISCONNECTED conn.connect.side_effect = lambda: conn._set_conn_state(ConnectionStates.CONNECTING) - assert cli._maybe_connect(0) is False + assert cli._init_connect(0) is True assert cli._conns[0] is conn @@ -127,8 +122,8 @@ def test_ready(mocker, cli, conn): def test_is_ready(mocker, cli, conn): - cli._maybe_connect(0) - cli._maybe_connect(1) + cli._init_connect(0) + cli._init_connect(1) # metadata refresh blocks ready nodes assert cli.is_ready(0) @@ -171,14 +166,14 @@ def test_close(mocker, cli, conn): assert conn.close.call_count == call_count # Single node close - cli._maybe_connect(0) + cli._init_connect(0) assert conn.close.call_count == call_count cli.close(0) call_count += 1 assert conn.close.call_count == call_count # All node close - cli._maybe_connect(1) + cli._init_connect(1) cli.close() # +2 close: node 1, node bootstrap (node 0 already closed) call_count += 2 @@ -190,7 +185,7 @@ def test_is_disconnected(cli, conn): conn.state = ConnectionStates.DISCONNECTED assert not cli.is_disconnected(0) - cli._maybe_connect(0) + cli._init_connect(0) assert cli.is_disconnected(0) conn.state = ConnectionStates.CONNECTING @@ -215,44 +210,40 @@ def test_send(cli, conn): assert isinstance(f.exception, Errors.NodeNotReadyError) conn.state = ConnectionStates.CONNECTED - cli._maybe_connect(0) + cli._init_connect(0) # ProduceRequest w/ 0 required_acks -> no response request = ProduceRequest[0](0, 0, []) assert request.expect_response() is False ret = cli.send(0, request) - conn.send.assert_called_with(request, blocking=False) + conn.send.assert_called_with(request, blocking=False, request_timeout_ms=None) assert isinstance(ret, Future) request = MetadataRequest[0]([]) cli.send(0, request) - conn.send.assert_called_with(request, blocking=False) + conn.send.assert_called_with(request, blocking=False, request_timeout_ms=None) def test_poll(mocker): metadata = mocker.patch.object(KafkaClient, '_maybe_refresh_metadata') + ifr_request_timeout = mocker.patch.object(KafkaClient, '_next_ifr_request_timeout_ms') _poll = mocker.patch.object(KafkaClient, '_poll') - ifrs = mocker.patch.object(KafkaClient, 'in_flight_request_count') - ifrs.return_value = 1 cli = KafkaClient(api_version=(0, 9)) # metadata timeout wins + ifr_request_timeout.return_value = float('inf') metadata.return_value = 1000 cli.poll() _poll.assert_called_with(1.0) # user timeout wins - cli.poll(250) + cli.poll(timeout_ms=250) _poll.assert_called_with(0.25) - # default is request_timeout_ms + # ifr request timeout wins + ifr_request_timeout.return_value = 30000 metadata.return_value = 1000000 cli.poll() - _poll.assert_called_with(cli.config['request_timeout_ms'] / 1000.0) - - # If no in-flight-requests, drop timeout to retry_backoff_ms - ifrs.return_value = 0 - cli.poll() - _poll.assert_called_with(cli.config['retry_backoff_ms'] / 1000.0) + _poll.assert_called_with(30.0) def test__poll(): @@ -270,7 +261,7 @@ def test_least_loaded_node(): def test_set_topics(mocker): request_update = mocker.patch.object(ClusterMetadata, 'request_update') request_update.side_effect = lambda: Future() - cli = KafkaClient(api_version=(0, 10)) + cli = KafkaClient(api_version=(0, 10, 0)) # replace 'empty' with 'non empty' request_update.reset_mock() @@ -309,25 +300,24 @@ def client(mocker): def test_maybe_refresh_metadata_ttl(mocker, client): client.cluster.ttl.return_value = 1234 - mocker.patch.object(KafkaClient, 'in_flight_request_count', return_value=1) client.poll(timeout_ms=12345678) client._poll.assert_called_with(1.234) def test_maybe_refresh_metadata_backoff(mocker, client): - mocker.patch.object(KafkaClient, 'in_flight_request_count', return_value=1) + mocker.patch.object(client, 'least_loaded_node', return_value=None) + mocker.patch.object(client, 'least_loaded_node_refresh_ms', return_value=4321) now = time.time() t = mocker.patch('time.time') t.return_value = now client.poll(timeout_ms=12345678) - client._poll.assert_called_with(2.222) # reconnect backoff + client._poll.assert_called_with(4.321) def test_maybe_refresh_metadata_in_progress(mocker, client): client._metadata_refresh_in_progress = True - mocker.patch.object(KafkaClient, 'in_flight_request_count', return_value=1) client.poll(timeout_ms=12345678) client._poll.assert_called_with(9999.999) # request_timeout_ms @@ -336,7 +326,6 @@ def test_maybe_refresh_metadata_in_progress(mocker, client): def test_maybe_refresh_metadata_update(mocker, client): mocker.patch.object(client, 'least_loaded_node', return_value='foobar') mocker.patch.object(client, '_can_send_request', return_value=True) - mocker.patch.object(KafkaClient, 'in_flight_request_count', return_value=1) send = mocker.patch.object(client, 'send') client.poll(timeout_ms=12345678) @@ -348,10 +337,9 @@ def test_maybe_refresh_metadata_update(mocker, client): def test_maybe_refresh_metadata_cant_send(mocker, client): mocker.patch.object(client, 'least_loaded_node', return_value='foobar') + mocker.patch.object(client, '_can_send_request', return_value=False) mocker.patch.object(client, '_can_connect', return_value=True) - mocker.patch.object(client, '_maybe_connect', return_value=True) - mocker.patch.object(client, 'maybe_connect', return_value=True) - mocker.patch.object(KafkaClient, 'in_flight_request_count', return_value=1) + mocker.patch.object(client, '_init_connect', return_value=True) now = time.time() t = mocker.patch('time.time') @@ -359,14 +347,14 @@ def test_maybe_refresh_metadata_cant_send(mocker, client): # first poll attempts connection client.poll(timeout_ms=12345678) - client._poll.assert_called_with(2.222) # reconnect backoff - client.maybe_connect.assert_called_once_with('foobar', wakeup=False) + client._poll.assert_called_with(12345.678) + client._init_connect.assert_called_once_with('foobar') # poll while connecting should not attempt a new connection client._connecting.add('foobar') client._can_connect.reset_mock() client.poll(timeout_ms=12345678) - client._poll.assert_called_with(2.222) # connection timeout (reconnect timeout) + client._poll.assert_called_with(12345.678) assert not client._can_connect.called assert not client._metadata_refresh_in_progress diff --git a/test/test_cluster.py b/test/test_cluster.py index f010c4f71..f0a2f83d6 100644 --- a/test/test_cluster.py +++ b/test/test_cluster.py @@ -1,8 +1,6 @@ # pylint: skip-file from __future__ import absolute_import -import pytest - from kafka.cluster import ClusterMetadata from kafka.protocol.metadata import MetadataResponse @@ -20,3 +18,117 @@ def test_empty_broker_list(): [], # empty brokers [(17, 'foo', []), (17, 'bar', [])])) # topics w/ error assert len(cluster.brokers()) == 2 + + +def test_metadata_v0(): + cluster = ClusterMetadata() + cluster.update_metadata(MetadataResponse[0]( + [(0, 'foo', 12), (1, 'bar', 34)], + [(0, 'topic-1', [(0, 0, 0, [0], [0])])])) + assert len(cluster.topics()) == 1 + assert cluster.controller is None + assert cluster.cluster_id is None + assert cluster._partitions['topic-1'][0].offline_replicas == [] + assert cluster._partitions['topic-1'][0].leader_epoch == -1 + + +def test_metadata_v1(): + cluster = ClusterMetadata() + cluster.update_metadata(MetadataResponse[1]( + [(0, 'foo', 12, 'rack-1'), (1, 'bar', 34, 'rack-2')], + 0, # controller_id + [(0, 'topic-1', False, [(0, 0, 0, [0], [0])])])) + assert len(cluster.topics()) == 1 + assert cluster.controller == cluster.broker_metadata(0) + assert cluster.cluster_id is None + assert cluster._partitions['topic-1'][0].offline_replicas == [] + assert cluster._partitions['topic-1'][0].leader_epoch == -1 + + +def test_metadata_v2(): + cluster = ClusterMetadata() + cluster.update_metadata(MetadataResponse[2]( + [(0, 'foo', 12, 'rack-1'), (1, 'bar', 34, 'rack-2')], + 'cluster-foo', # cluster_id + 0, # controller_id + [(0, 'topic-1', False, [(0, 0, 0, [0], [0])])])) + assert len(cluster.topics()) == 1 + assert cluster.controller == cluster.broker_metadata(0) + assert cluster.cluster_id == 'cluster-foo' + assert cluster._partitions['topic-1'][0].offline_replicas == [] + assert cluster._partitions['topic-1'][0].leader_epoch == -1 + + +def test_metadata_v3(): + cluster = ClusterMetadata() + cluster.update_metadata(MetadataResponse[3]( + 0, # throttle_time_ms + [(0, 'foo', 12, 'rack-1'), (1, 'bar', 34, 'rack-2')], + 'cluster-foo', # cluster_id + 0, # controller_id + [(0, 'topic-1', False, [(0, 0, 0, [0], [0])])])) + assert len(cluster.topics()) == 1 + assert cluster.controller == cluster.broker_metadata(0) + assert cluster.cluster_id == 'cluster-foo' + assert cluster._partitions['topic-1'][0].offline_replicas == [] + assert cluster._partitions['topic-1'][0].leader_epoch == -1 + + +def test_metadata_v4(): + cluster = ClusterMetadata() + cluster.update_metadata(MetadataResponse[4]( + 0, # throttle_time_ms + [(0, 'foo', 12, 'rack-1'), (1, 'bar', 34, 'rack-2')], + 'cluster-foo', # cluster_id + 0, # controller_id + [(0, 'topic-1', False, [(0, 0, 0, [0], [0])])])) + assert len(cluster.topics()) == 1 + assert cluster.controller == cluster.broker_metadata(0) + assert cluster.cluster_id == 'cluster-foo' + assert cluster._partitions['topic-1'][0].offline_replicas == [] + assert cluster._partitions['topic-1'][0].leader_epoch == -1 + + +def test_metadata_v5(): + cluster = ClusterMetadata() + cluster.update_metadata(MetadataResponse[5]( + 0, # throttle_time_ms + [(0, 'foo', 12, 'rack-1'), (1, 'bar', 34, 'rack-2')], + 'cluster-foo', # cluster_id + 0, # controller_id + [(0, 'topic-1', False, [(0, 0, 0, [0], [0], [12])])])) + assert len(cluster.topics()) == 1 + assert cluster.controller == cluster.broker_metadata(0) + assert cluster.cluster_id == 'cluster-foo' + assert cluster._partitions['topic-1'][0].offline_replicas == [12] + assert cluster._partitions['topic-1'][0].leader_epoch == -1 + + +def test_metadata_v6(): + cluster = ClusterMetadata() + cluster.update_metadata(MetadataResponse[6]( + 0, # throttle_time_ms + [(0, 'foo', 12, 'rack-1'), (1, 'bar', 34, 'rack-2')], + 'cluster-foo', # cluster_id + 0, # controller_id + [(0, 'topic-1', False, [(0, 0, 0, [0], [0], [12])])])) + assert len(cluster.topics()) == 1 + assert cluster.controller == cluster.broker_metadata(0) + assert cluster.cluster_id == 'cluster-foo' + assert cluster._partitions['topic-1'][0].offline_replicas == [12] + assert cluster._partitions['topic-1'][0].leader_epoch == -1 + + +def test_metadata_v7(): + cluster = ClusterMetadata() + cluster.update_metadata(MetadataResponse[7]( + 0, # throttle_time_ms + [(0, 'foo', 12, 'rack-1'), (1, 'bar', 34, 'rack-2')], + 'cluster-foo', # cluster_id + 0, # controller_id + [(0, 'topic-1', False, [(0, 0, 0, 0, [0], [0], [12])])])) + assert len(cluster.topics()) == 1 + assert cluster.controller == cluster.broker_metadata(0) + assert cluster.cluster_id == 'cluster-foo' + assert cluster._partitions['topic-1'][0].offline_replicas == [12] + assert cluster._partitions['topic-1'][0].leader_epoch == 0 diff --git a/test/test_codec.py b/test/test_codec.py index e05707451..24159c253 100644 --- a/test/test_codec.py +++ b/test/test_codec.py @@ -39,12 +39,14 @@ def test_snappy_detect_xerial(): _detect_xerial_stream = kafka1.codec._detect_xerial_stream header = b'\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01Some extra bytes' + redpanda_header = b'\x82SNAPPY\x00\x01\x00\x00\x00\x01\x00\x00\x00Some extra bytes' false_header = b'\x01SNAPPY\x00\x00\x00\x01\x00\x00\x00\x01' default_snappy = snappy_encode(b'foobar' * 50) random_snappy = snappy_encode(b'SNAPPY' * 50, xerial_compatible=False) short_data = b'\x01\x02\x03\x04' assert _detect_xerial_stream(header) is True + assert _detect_xerial_stream(redpanda_header) is True assert _detect_xerial_stream(b'') is False assert _detect_xerial_stream(b'\x00') is False assert _detect_xerial_stream(false_header) is False diff --git a/test/test_conn.py b/test/test_conn.py index 966f7b34d..ea88fd04c 100644 --- a/test/test_conn.py +++ b/test/test_conn.py @@ -4,16 +4,27 @@ from errno import EALREADY, EINPROGRESS, EISCONN, ECONNRESET import socket -import mock +try: + from unittest import mock +except ImportError: + import mock import pytest from kafka.conn import BrokerConnection, ConnectionStates, collect_hosts from kafka.protocol.api import RequestHeader +from kafka.protocol.group import HeartbeatResponse from kafka.protocol.metadata import MetadataRequest from kafka.protocol.produce import ProduceRequest import kafka.errors as Errors +from kafka.vendor import six + +if six.PY2: + ConnectionError = socket.error + TimeoutError = socket.error + BlockingIOError = Exception + @pytest.fixture def dns_lookup(mocker): @@ -26,13 +37,16 @@ def dns_lookup(mocker): def _socket(mocker): socket = mocker.MagicMock() socket.connect_ex.return_value = 0 + socket.send.side_effect = lambda d: len(d) + socket.recv.side_effect = BlockingIOError("mocked recv") mocker.patch('socket.socket', return_value=socket) return socket @pytest.fixture -def conn(_socket, dns_lookup): +def conn(_socket, dns_lookup, mocker): conn = BrokerConnection('localhost', 9092, socket.AF_INET) + mocker.patch.object(conn, '_try_api_versions_check', return_value=True) return conn @@ -80,15 +94,35 @@ def test_blacked_out(conn): assert conn.blacked_out() is True -def test_connection_delay(conn): +def test_connection_delay(conn, mocker): + mocker.patch.object(conn, '_reconnect_jitter_pct', return_value=1.0) with mock.patch("time.time", return_value=1000): conn.last_attempt = 1000 assert conn.connection_delay() == conn.config['reconnect_backoff_ms'] conn.state = ConnectionStates.CONNECTING - assert conn.connection_delay() == float('inf') + assert conn.connection_delay() == conn.config['reconnect_backoff_ms'] conn.state = ConnectionStates.CONNECTED assert conn.connection_delay() == float('inf') + del conn._gai[:] + conn._update_reconnect_backoff() + conn.state = ConnectionStates.DISCONNECTED + assert conn.connection_delay() == 1.0 * conn.config['reconnect_backoff_ms'] + conn.state = ConnectionStates.CONNECTING + assert conn.connection_delay() == 1.0 * conn.config['reconnect_backoff_ms'] + + conn._update_reconnect_backoff() + conn.state = ConnectionStates.DISCONNECTED + assert conn.connection_delay() == 2.0 * conn.config['reconnect_backoff_ms'] + conn.state = ConnectionStates.CONNECTING + assert conn.connection_delay() == 2.0 * conn.config['reconnect_backoff_ms'] + + conn._update_reconnect_backoff() + conn.state = ConnectionStates.DISCONNECTED + assert conn.connection_delay() == 4.0 * conn.config['reconnect_backoff_ms'] + conn.state = ConnectionStates.CONNECTING + assert conn.connection_delay() == 4.0 * conn.config['reconnect_backoff_ms'] + def test_connected(conn): assert conn.connected() is False @@ -196,12 +230,13 @@ def test_recv_disconnected(_socket, conn): conn.send(req) # Empty data on recv means the socket is disconnected + _socket.recv.side_effect = None _socket.recv.return_value = b'' # Attempt to receive should mark connection as disconnected - assert conn.connected() + assert conn.connected(), 'Not connected: %s' % conn.state conn.recv() - assert conn.disconnected() + assert conn.disconnected(), 'Not disconnected: %s' % conn.state def test_recv(_socket, conn): @@ -327,16 +362,42 @@ def test_requests_timed_out(conn): # No in-flight requests, not timed out assert not conn.requests_timed_out() - # Single request, timestamp = now (0) - conn.in_flight_requests[0] = ('foo', 0) + # Single request, timeout_at > now (0) + conn.in_flight_requests[0] = ('foo', 0, 1) assert not conn.requests_timed_out() # Add another request w/ timestamp > request_timeout ago request_timeout = conn.config['request_timeout_ms'] expired_timestamp = 0 - request_timeout - 1 - conn.in_flight_requests[1] = ('bar', expired_timestamp) + conn.in_flight_requests[1] = ('bar', 0, expired_timestamp) assert conn.requests_timed_out() # Drop the expired request and we should be good to go again conn.in_flight_requests.pop(1) assert not conn.requests_timed_out() + + +def test_maybe_throttle(conn): + assert conn.state is ConnectionStates.DISCONNECTED + assert not conn.throttled() + + conn.state = ConnectionStates.CONNECTED + assert not conn.throttled() + + # No throttle_time_ms attribute + conn._maybe_throttle(HeartbeatResponse[0](error_code=0)) + assert not conn.throttled() + + with mock.patch("time.time", return_value=1000) as time: + # server-side throttling in v1.0 + conn.config['api_version'] = (1, 0) + conn._maybe_throttle(HeartbeatResponse[1](throttle_time_ms=1000, error_code=0)) + assert not conn.throttled() + + # client-side throttling in v2.0 + conn.config['api_version'] = (2, 0) + conn._maybe_throttle(HeartbeatResponse[2](throttle_time_ms=1000, error_code=0)) + assert conn.throttled() + + time.return_value = 3000 + assert not conn.throttled() diff --git a/test/test_consumer.py b/test/test_consumer.py index 436fe55c0..8186125df 100644 --- a/test/test_consumer.py +++ b/test/test_consumer.py @@ -18,7 +18,7 @@ def test_request_timeout_larger_than_connections_max_idle_ms_raises(self): KafkaConsumer(bootstrap_servers='localhost:9092', api_version=(0, 9), request_timeout_ms=50000, connections_max_idle_ms=40000) def test_subscription_copy(self): - consumer = KafkaConsumer('foo', api_version=(0, 10)) + consumer = KafkaConsumer('foo', api_version=(0, 10, 0)) sub = consumer.subscription() assert sub is not consumer.subscription() assert sub == set(['foo']) diff --git a/test/test_consumer_group.py b/test/test_consumer_group.py index 58dc7ebf9..bc04eed48 100644 --- a/test/test_consumer_group.py +++ b/test/test_consumer_group.py @@ -23,7 +23,7 @@ def test_consumer(kafka_broker, topic): # The `topic` fixture is included because # 0.8.2 brokers need a topic to function well consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker)) - consumer.poll(500) + consumer.poll(timeout_ms=500) assert len(consumer._client._conns) > 0 node_id = list(consumer._client._conns.keys())[0] assert consumer._client._conns[node_id].state is ConnectionStates.CONNECTED @@ -34,7 +34,7 @@ def test_consumer(kafka_broker, topic): def test_consumer_topics(kafka_broker, topic): consumer = KafkaConsumer(bootstrap_servers=get_connect_str(kafka_broker)) # Necessary to drive the IO - consumer.poll(500) + consumer.poll(timeout_ms=500) assert topic in consumer.topics() assert len(consumer.partitions_for_topic(topic)) > 0 consumer.close() @@ -56,9 +56,11 @@ def consumer_thread(i): consumers[i] = KafkaConsumer(topic, bootstrap_servers=connect_str, group_id=group_id, + client_id="consumer_thread-%s" % i, + api_version_auto_timeout_ms=30000, heartbeat_interval_ms=500) while not stop[i].is_set(): - for tp, records in six.itervalues(consumers[i].poll(100)): + for tp, records in six.itervalues(consumers[i].poll(timeout_ms=200)): messages[i][tp].extend(records) consumers[i].close() consumers[i] = None @@ -155,6 +157,7 @@ def test_heartbeat_thread(kafka_broker, topic): consumer = KafkaConsumer(topic, bootstrap_servers=get_connect_str(kafka_broker), group_id=group_id, + api_version_auto_timeout_ms=30000, heartbeat_interval_ms=500) # poll until we have joined group / have assignment diff --git a/test/test_consumer_integration.py b/test/test_consumer_integration.py index 90b7ed203..af8ec6829 100644 --- a/test/test_consumer_integration.py +++ b/test/test_consumer_integration.py @@ -1,7 +1,10 @@ import logging import time -from mock import patch +try: + from unittest.mock import patch, ANY +except ImportError: + from mock import patch, ANY import pytest from kafka.vendor.six.moves import range @@ -13,6 +16,7 @@ @pytest.mark.skipif(not env_kafka_version(), reason="No KAFKA_VERSION set") +@pytest.mark.skipif(env_kafka_version()[:2] > (2, 6, 0), reason="KAFKA_VERSION newer than max inferred version") def test_kafka_version_infer(kafka_consumer_factory): consumer = kafka_consumer_factory() actual_ver_major_minor = env_kafka_version()[:2] @@ -26,7 +30,7 @@ def test_kafka_version_infer(kafka_consumer_factory): @pytest.mark.skipif(not env_kafka_version(), reason="No KAFKA_VERSION set") def test_kafka_consumer(kafka_consumer_factory, send_messages): """Test KafkaConsumer""" - consumer = kafka_consumer_factory(auto_offset_reset='earliest') + consumer = kafka_consumer_factory(auto_offset_reset='earliest', consumer_timeout_ms=2000) send_messages(range(0, 100), partition=0) send_messages(range(0, 100), partition=1) cnt = 0 @@ -257,9 +261,10 @@ def test_kafka_consumer_offsets_search_many_partitions(kafka_consumer, kafka_pro tp1: send_time }) + leader_epoch = ANY if env_kafka_version() >= (2, 1) else -1 assert offsets == { - tp0: OffsetAndTimestamp(p0msg.offset, send_time), - tp1: OffsetAndTimestamp(p1msg.offset, send_time) + tp0: OffsetAndTimestamp(p0msg.offset, send_time, leader_epoch), + tp1: OffsetAndTimestamp(p1msg.offset, send_time, leader_epoch) } offsets = consumer.beginning_offsets([tp0, tp1]) @@ -275,6 +280,7 @@ def test_kafka_consumer_offsets_search_many_partitions(kafka_consumer, kafka_pro } +@pytest.mark.skipif(not env_kafka_version(), reason="No KAFKA_VERSION set") @pytest.mark.skipif(env_kafka_version() >= (0, 10, 1), reason="Requires KAFKA_VERSION < 0.10.1") def test_kafka_consumer_offsets_for_time_old(kafka_consumer, topic): consumer = kafka_consumer diff --git a/test/test_coordinator.py b/test/test_coordinator.py index a35cdd1a0..09422790e 100644 --- a/test/test_coordinator.py +++ b/test/test_coordinator.py @@ -17,6 +17,7 @@ import kafka.errors as Errors from kafka.future import Future from kafka.metrics import Metrics +from kafka.protocol.broker_api_versions import BROKER_API_VERSIONS from kafka.protocol.commit import ( OffsetCommitRequest, OffsetCommitResponse, OffsetFetchRequest, OffsetFetchResponse) @@ -41,8 +42,9 @@ def test_init(client, coordinator): @pytest.mark.parametrize("api_version", [(0, 8, 0), (0, 8, 1), (0, 8, 2), (0, 9)]) -def test_autocommit_enable_api_version(client, api_version): - coordinator = ConsumerCoordinator(client, SubscriptionState(), +def test_autocommit_enable_api_version(conn, api_version): + coordinator = ConsumerCoordinator(KafkaClient(api_version=api_version), + SubscriptionState(), Metrics(), enable_auto_commit=True, session_timeout_ms=30000, # session_timeout_ms and max_poll_interval_ms @@ -86,8 +88,13 @@ def test_group_protocols(coordinator): @pytest.mark.parametrize('api_version', [(0, 8, 0), (0, 8, 1), (0, 8, 2), (0, 9)]) -def test_pattern_subscription(coordinator, api_version): - coordinator.config['api_version'] = api_version +def test_pattern_subscription(conn, api_version): + coordinator = ConsumerCoordinator(KafkaClient(api_version=api_version), + SubscriptionState(), + Metrics(), + api_version=api_version, + session_timeout_ms=10000, + max_poll_interval_ms=10000) coordinator._subscription.subscribe(pattern='foo') assert coordinator._subscription.subscription == set([]) assert coordinator._metadata_snapshot == coordinator._build_metadata_snapshot(coordinator._subscription, {}) @@ -223,13 +230,13 @@ def test_need_rejoin(coordinator): def test_refresh_committed_offsets_if_needed(mocker, coordinator): mocker.patch.object(ConsumerCoordinator, 'fetch_committed_offsets', return_value = { - TopicPartition('foobar', 0): OffsetAndMetadata(123, b''), - TopicPartition('foobar', 1): OffsetAndMetadata(234, b'')}) + TopicPartition('foobar', 0): OffsetAndMetadata(123, '', -1), + TopicPartition('foobar', 1): OffsetAndMetadata(234, '', -1)}) coordinator._subscription.assign_from_user([TopicPartition('foobar', 0)]) assert coordinator._subscription.needs_fetch_committed_offsets is True coordinator.refresh_committed_offsets_if_needed() assignment = coordinator._subscription.assignment - assert assignment[TopicPartition('foobar', 0)].committed == OffsetAndMetadata(123, b'') + assert assignment[TopicPartition('foobar', 0)].committed == OffsetAndMetadata(123, '', -1) assert TopicPartition('foobar', 1) not in assignment assert coordinator._subscription.needs_fetch_committed_offsets is False @@ -296,8 +303,8 @@ def test_close(mocker, coordinator): @pytest.fixture def offsets(): return { - TopicPartition('foobar', 0): OffsetAndMetadata(123, b''), - TopicPartition('foobar', 1): OffsetAndMetadata(234, b''), + TopicPartition('foobar', 0): OffsetAndMetadata(123, '', -1), + TopicPartition('foobar', 1): OffsetAndMetadata(234, '', -1), } @@ -432,11 +439,15 @@ def test_send_offset_commit_request_fail(mocker, patched_coord, offsets): @pytest.mark.parametrize('api_version,req_type', [ ((0, 8, 1), OffsetCommitRequest[0]), ((0, 8, 2), OffsetCommitRequest[1]), - ((0, 9), OffsetCommitRequest[2])]) + ((0, 9), OffsetCommitRequest[2]), + ((0, 11), OffsetCommitRequest[3]), + ((2, 0), OffsetCommitRequest[4]), + ((2, 1), OffsetCommitRequest[6]), +]) def test_send_offset_commit_request_versions(patched_coord, offsets, api_version, req_type): expect_node = 0 - patched_coord.config['api_version'] = api_version + patched_coord._client._api_versions = BROKER_API_VERSIONS[api_version] patched_coord._send_offset_commit_request(offsets) (node, request), _ = patched_coord._client.send.call_args @@ -492,13 +503,27 @@ def test_send_offset_commit_request_success(mocker, patched_coord, offsets): Errors.InvalidTopicError, False), (OffsetCommitResponse[0]([('foobar', [(0, 29), (1, 29)])]), Errors.TopicAuthorizationFailedError, False), + (OffsetCommitResponse[0]([('foobar', [(0, 0), (1, 0)])]), + None, False), + (OffsetCommitResponse[1]([('foobar', [(0, 0), (1, 0)])]), + None, False), + (OffsetCommitResponse[2]([('foobar', [(0, 0), (1, 0)])]), + None, False), + (OffsetCommitResponse[3](0, [('foobar', [(0, 0), (1, 0)])]), + None, False), + (OffsetCommitResponse[4](0, [('foobar', [(0, 0), (1, 0)])]), + None, False), + (OffsetCommitResponse[5](0, [('foobar', [(0, 0), (1, 0)])]), + None, False), + (OffsetCommitResponse[6](0, [('foobar', [(0, 0), (1, 0)])]), + None, False), ]) def test_handle_offset_commit_response(mocker, patched_coord, offsets, response, error, dead): future = Future() patched_coord._handle_offset_commit_response(offsets, future, time.time(), response) - assert isinstance(future.exception, error) + assert isinstance(future.exception, error) if error else True assert patched_coord.coordinator_id is (None if dead else 0) @@ -527,12 +552,17 @@ def test_send_offset_fetch_request_fail(mocker, patched_coord, partitions): @pytest.mark.parametrize('api_version,req_type', [ ((0, 8, 1), OffsetFetchRequest[0]), ((0, 8, 2), OffsetFetchRequest[1]), - ((0, 9), OffsetFetchRequest[1])]) + ((0, 9), OffsetFetchRequest[1]), + ((0, 10, 2), OffsetFetchRequest[2]), + ((0, 11), OffsetFetchRequest[3]), + ((2, 0), OffsetFetchRequest[4]), + ((2, 1), OffsetFetchRequest[5]), +]) def test_send_offset_fetch_request_versions(patched_coord, partitions, api_version, req_type): # assuming fixture sets coordinator=0, least_loaded_node=1 expect_node = 0 - patched_coord.config['api_version'] = api_version + patched_coord._client._api_versions = BROKER_API_VERSIONS[api_version] patched_coord._send_offset_fetch_request(partitions) (node, request), _ = patched_coord._client.send.call_args @@ -564,17 +594,27 @@ def test_send_offset_fetch_request_success(patched_coord, partitions): @pytest.mark.parametrize('response,error,dead', [ - (OffsetFetchResponse[0]([('foobar', [(0, 123, b'', 14), (1, 234, b'', 14)])]), + (OffsetFetchResponse[0]([('foobar', [(0, 123, '', 14), (1, 234, '', 14)])]), Errors.GroupLoadInProgressError, False), - (OffsetFetchResponse[0]([('foobar', [(0, 123, b'', 16), (1, 234, b'', 16)])]), + (OffsetFetchResponse[0]([('foobar', [(0, 123, '', 16), (1, 234, '', 16)])]), Errors.NotCoordinatorForGroupError, True), - (OffsetFetchResponse[0]([('foobar', [(0, 123, b'', 25), (1, 234, b'', 25)])]), + (OffsetFetchResponse[0]([('foobar', [(0, 123, '', 25), (1, 234, '', 25)])]), Errors.UnknownMemberIdError, False), - (OffsetFetchResponse[0]([('foobar', [(0, 123, b'', 22), (1, 234, b'', 22)])]), + (OffsetFetchResponse[0]([('foobar', [(0, 123, '', 22), (1, 234, '', 22)])]), Errors.IllegalGenerationError, False), - (OffsetFetchResponse[0]([('foobar', [(0, 123, b'', 29), (1, 234, b'', 29)])]), + (OffsetFetchResponse[0]([('foobar', [(0, 123, '', 29), (1, 234, '', 29)])]), Errors.TopicAuthorizationFailedError, False), - (OffsetFetchResponse[0]([('foobar', [(0, 123, b'', 0), (1, 234, b'', 0)])]), + (OffsetFetchResponse[0]([('foobar', [(0, 123, '', 0), (1, 234, '', 0)])]), + None, False), + (OffsetFetchResponse[1]([('foobar', [(0, 123, '', 0), (1, 234, '', 0)])]), + None, False), + (OffsetFetchResponse[2]([('foobar', [(0, 123, '', 0), (1, 234, '', 0)])], 0), + None, False), + (OffsetFetchResponse[3](0, [('foobar', [(0, 123, '', 0), (1, 234, '', 0)])], 0), + None, False), + (OffsetFetchResponse[4](0, [('foobar', [(0, 123, '', 0), (1, 234, '', 0)])], 0), + None, False), + (OffsetFetchResponse[5](0, [('foobar', [(0, 123, -1, '', 0), (1, 234, -1, '', 0)])], 0), None, False), ]) def test_handle_offset_fetch_response(patched_coord, offsets, diff --git a/test/test_fetcher.py b/test/test_fetcher.py index 697f8be1f..479f6e22b 100644 --- a/test/test_fetcher.py +++ b/test/test_fetcher.py @@ -1,5 +1,6 @@ # pylint: skip-file from __future__ import absolute_import +import logging import pytest @@ -9,24 +10,26 @@ from kafka.client_async import KafkaClient from kafka.consumer.fetcher import ( - CompletedFetch, ConsumerRecord, Fetcher, NoOffsetForPartitionError + CompletedFetch, ConsumerRecord, Fetcher ) from kafka.consumer.subscription_state import SubscriptionState +import kafka.errors as Errors from kafka.future import Future from kafka.metrics import Metrics +from kafka.protocol.broker_api_versions import BROKER_API_VERSIONS from kafka.protocol.fetch import FetchRequest, FetchResponse -from kafka.protocol.offset import OffsetResponse +from kafka.protocol.list_offsets import ListOffsetsResponse from kafka.errors import ( StaleMetadata, LeaderNotAvailableError, NotLeaderForPartitionError, UnknownTopicOrPartitionError, OffsetOutOfRangeError ) from kafka.record.memory_records import MemoryRecordsBuilder, MemoryRecords -from kafka.structs import OffsetAndMetadata, TopicPartition +from kafka.structs import OffsetAndMetadata, OffsetAndTimestamp, TopicPartition @pytest.fixture -def client(mocker): - return mocker.Mock(spec=KafkaClient(bootstrap_servers=(), api_version=(0, 9))) +def client(): + return KafkaClient(bootstrap_servers=(), api_version=(0, 9)) @pytest.fixture @@ -76,9 +79,20 @@ def test_send_fetches(fetcher, topic, mocker): ])]) ] - mocker.patch.object(fetcher, '_create_fetch_requests', - return_value=dict(enumerate(fetch_requests))) + def build_fetch_offsets(request): + fetch_offsets = {} + for topic, partitions in request.topics: + for partition_data in partitions: + partition, offset = partition_data[:2] + fetch_offsets[TopicPartition(topic, partition)] = offset + return fetch_offsets + mocker.patch.object( + fetcher, '_create_fetch_requests', + return_value=(dict(enumerate(map(lambda r: (r, build_fetch_offsets(r)), fetch_requests))))) + + mocker.patch.object(fetcher._client, 'ready', return_value=True) + mocker.patch.object(fetcher._client, 'send') ret = fetcher.send_fetches() for node, request in enumerate(fetch_requests): fetcher._client.send.assert_any_call(node, request, wakeup=False) @@ -89,14 +103,15 @@ def test_send_fetches(fetcher, topic, mocker): ((0, 10, 1), 3), ((0, 10, 0), 2), ((0, 9), 1), - ((0, 8), 0) + ((0, 8, 2), 0) ]) def test_create_fetch_requests(fetcher, mocker, api_version, fetch_version): - fetcher._client.in_flight_request_count.return_value = 0 - fetcher.config['api_version'] = api_version + fetcher._client._api_versions = BROKER_API_VERSIONS[api_version] + mocker.patch.object(fetcher._client.cluster, "leader_for_partition", return_value=0) + mocker.patch.object(fetcher._client.cluster, "leader_epoch_for_partition", return_value=0) by_node = fetcher._create_fetch_requests() - requests = by_node.values() - assert all([isinstance(r, FetchRequest[fetch_version]) for r in requests]) + requests_and_offsets = by_node.values() + assert set([r.API_VERSION for (r, _offsets) in requests_and_offsets]) == set([fetch_version]) def test_update_fetch_positions(fetcher, topic, mocker): @@ -124,7 +139,7 @@ def test_update_fetch_positions(fetcher, topic, mocker): fetcher._reset_offset.reset_mock() fetcher._subscriptions.need_offset_reset(partition) fetcher._subscriptions.assignment[partition].awaiting_reset = False - fetcher._subscriptions.assignment[partition].committed = OffsetAndMetadata(123, b'') + fetcher._subscriptions.assignment[partition].committed = OffsetAndMetadata(123, '', -1) mocker.patch.object(fetcher._subscriptions, 'seek') fetcher.update_fetch_positions([partition]) assert fetcher._reset_offset.call_count == 0 @@ -138,15 +153,15 @@ def test__reset_offset(fetcher, mocker): fetcher._subscriptions.need_offset_reset(tp) mocked = mocker.patch.object(fetcher, '_retrieve_offsets') - mocked.return_value = {tp: (1001, None)} + mocked.return_value = {tp: OffsetAndTimestamp(1001, None, -1)} fetcher._reset_offset(tp) assert not fetcher._subscriptions.assignment[tp].awaiting_reset - assert fetcher._subscriptions.assignment[tp].position == 1001 + assert fetcher._subscriptions.assignment[tp].position.offset == 1001 -def test__send_offset_requests(fetcher, mocker): - tp = TopicPartition("topic_send_offset", 1) - mocked_send = mocker.patch.object(fetcher, "_send_offset_request") +def test__send_list_offsets_requests(fetcher, mocker): + tp = TopicPartition("topic_send_list_offsets", 1) + mocked_send = mocker.patch.object(fetcher, "_send_list_offsets_request") send_futures = [] def send_side_effect(*args, **kw): @@ -161,21 +176,22 @@ def send_side_effect(*args, **kw): # always as available mocked_leader.side_effect = itertools.chain( [None, -1], itertools.cycle([0])) + mocker.patch.object(fetcher._client.cluster, "leader_epoch_for_partition", return_value=0) # Leader == None - fut = fetcher._send_offset_requests({tp: 0}) + fut = fetcher._send_list_offsets_requests({tp: 0}) assert fut.failed() assert isinstance(fut.exception, StaleMetadata) assert not mocked_send.called # Leader == -1 - fut = fetcher._send_offset_requests({tp: 0}) + fut = fetcher._send_list_offsets_requests({tp: 0}) assert fut.failed() assert isinstance(fut.exception, LeaderNotAvailableError) assert not mocked_send.called # Leader == 0, send failed - fut = fetcher._send_offset_requests({tp: 0}) + fut = fetcher._send_list_offsets_requests({tp: 0}) assert not fut.is_done assert mocked_send.called # Check that we bound the futures correctly to chain failure @@ -184,7 +200,7 @@ def send_side_effect(*args, **kw): assert isinstance(fut.exception, NotLeaderForPartitionError) # Leader == 0, send success - fut = fetcher._send_offset_requests({tp: 0}) + fut = fetcher._send_list_offsets_requests({tp: 0}) assert not fut.is_done assert mocked_send.called # Check that we bound the futures correctly to chain success @@ -193,12 +209,12 @@ def send_side_effect(*args, **kw): assert fut.value == {tp: (10, 10000)} -def test__send_offset_requests_multiple_nodes(fetcher, mocker): - tp1 = TopicPartition("topic_send_offset", 1) - tp2 = TopicPartition("topic_send_offset", 2) - tp3 = TopicPartition("topic_send_offset", 3) - tp4 = TopicPartition("topic_send_offset", 4) - mocked_send = mocker.patch.object(fetcher, "_send_offset_request") +def test__send_list_offsets_requests_multiple_nodes(fetcher, mocker): + tp1 = TopicPartition("topic_send_list_offsets", 1) + tp2 = TopicPartition("topic_send_list_offsets", 2) + tp3 = TopicPartition("topic_send_list_offsets", 3) + tp4 = TopicPartition("topic_send_list_offsets", 4) + mocked_send = mocker.patch.object(fetcher, "_send_list_offsets_request") send_futures = [] def send_side_effect(node_id, timestamps): @@ -210,10 +226,11 @@ def send_side_effect(node_id, timestamps): mocked_leader = mocker.patch.object( fetcher._client.cluster, "leader_for_partition") mocked_leader.side_effect = itertools.cycle([0, 1]) + mocker.patch.object(fetcher._client.cluster, "leader_epoch_for_partition", return_value=0) # -- All node succeeded case tss = OrderedDict([(tp1, 0), (tp2, 0), (tp3, 0), (tp4, 0)]) - fut = fetcher._send_offset_requests(tss) + fut = fetcher._send_list_offsets_requests(tss) assert not fut.is_done assert mocked_send.call_count == 2 @@ -227,8 +244,8 @@ def send_side_effect(node_id, timestamps): else: second_future = f assert req_by_node == { - 0: {tp1: 0, tp3: 0}, - 1: {tp2: 0, tp4: 0} + 0: {tp1: (0, -1), tp3: (0, -1)}, + 1: {tp2: (0, -1), tp4: (0, -1)} } # We only resolved 1 future so far, so result future is not yet ready @@ -239,7 +256,7 @@ def send_side_effect(node_id, timestamps): # -- First succeeded second not del send_futures[:] - fut = fetcher._send_offset_requests(tss) + fut = fetcher._send_list_offsets_requests(tss) assert len(send_futures) == 2 send_futures[0][2].success({tp1: (11, 1001)}) send_futures[1][2].failure(UnknownTopicOrPartitionError(tp1)) @@ -248,7 +265,7 @@ def send_side_effect(node_id, timestamps): # -- First fails second succeeded del send_futures[:] - fut = fetcher._send_offset_requests(tss) + fut = fetcher._send_list_offsets_requests(tss) assert len(send_futures) == 2 send_futures[0][2].failure(UnknownTopicOrPartitionError(tp1)) send_futures[1][2].success({tp1: (11, 1001)}) @@ -256,49 +273,93 @@ def send_side_effect(node_id, timestamps): assert isinstance(fut.exception, UnknownTopicOrPartitionError) -def test__handle_offset_response(fetcher, mocker): +def test__handle_list_offsets_response_v1(fetcher, mocker): # Broker returns UnsupportedForMessageFormatError, will omit partition fut = Future() - res = OffsetResponse[1]([ + res = ListOffsetsResponse[1]([ ("topic", [(0, 43, -1, -1)]), ("topic", [(1, 0, 1000, 9999)]) ]) - fetcher._handle_offset_response(fut, res) + fetcher._handle_list_offsets_response(fut, res) assert fut.succeeded() - assert fut.value == {TopicPartition("topic", 1): (9999, 1000)} + assert fut.value == {TopicPartition("topic", 1): OffsetAndTimestamp(9999, 1000, -1)} # Broker returns NotLeaderForPartitionError fut = Future() - res = OffsetResponse[1]([ + res = ListOffsetsResponse[1]([ ("topic", [(0, 6, -1, -1)]), ]) - fetcher._handle_offset_response(fut, res) + fetcher._handle_list_offsets_response(fut, res) assert fut.failed() assert isinstance(fut.exception, NotLeaderForPartitionError) # Broker returns UnknownTopicOrPartitionError fut = Future() - res = OffsetResponse[1]([ + res = ListOffsetsResponse[1]([ ("topic", [(0, 3, -1, -1)]), ]) - fetcher._handle_offset_response(fut, res) + fetcher._handle_list_offsets_response(fut, res) assert fut.failed() assert isinstance(fut.exception, UnknownTopicOrPartitionError) # Broker returns many errors and 1 result # Will fail on 1st error and return fut = Future() - res = OffsetResponse[1]([ + res = ListOffsetsResponse[1]([ ("topic", [(0, 43, -1, -1)]), ("topic", [(1, 6, -1, -1)]), ("topic", [(2, 3, -1, -1)]), ("topic", [(3, 0, 1000, 9999)]) ]) - fetcher._handle_offset_response(fut, res) + fetcher._handle_list_offsets_response(fut, res) assert fut.failed() assert isinstance(fut.exception, NotLeaderForPartitionError) +def test__handle_list_offsets_response_v2_v3(fetcher, mocker): + # including a throttle_time shouldnt cause issues + fut = Future() + res = ListOffsetsResponse[2]( + 123, # throttle_time_ms + [("topic", [(0, 0, 1000, 9999)]) + ]) + fetcher._handle_list_offsets_response(fut, res) + assert fut.succeeded() + assert fut.value == {TopicPartition("topic", 0): OffsetAndTimestamp(9999, 1000, -1)} + + # v3 response is the same format + fut = Future() + res = ListOffsetsResponse[3]( + 123, # throttle_time_ms + [("topic", [(0, 0, 1000, 9999)]) + ]) + fetcher._handle_list_offsets_response(fut, res) + assert fut.succeeded() + assert fut.value == {TopicPartition("topic", 0): OffsetAndTimestamp(9999, 1000, -1)} + + +def test__handle_list_offsets_response_v4_v5(fetcher, mocker): + # includes leader_epoch + fut = Future() + res = ListOffsetsResponse[4]( + 123, # throttle_time_ms + [("topic", [(0, 0, 1000, 9999, 1234)]) + ]) + fetcher._handle_list_offsets_response(fut, res) + assert fut.succeeded() + assert fut.value == {TopicPartition("topic", 0): OffsetAndTimestamp(9999, 1000, 1234)} + + # v5 response is the same format + fut = Future() + res = ListOffsetsResponse[5]( + 123, # throttle_time_ms + [("topic", [(0, 0, 1000, 9999, 1234)]) + ]) + fetcher._handle_list_offsets_response(fut, res) + assert fut.succeeded() + assert fut.value == {TopicPartition("topic", 0): OffsetAndTimestamp(9999, 1000, 1234)} + + def test_fetched_records(fetcher, topic, mocker): fetcher.config['check_crcs'] = False tp = TopicPartition(topic, 0) @@ -318,19 +379,15 @@ def test_fetched_records(fetcher, topic, mocker): assert partial is False -@pytest.mark.parametrize(("fetch_request", "fetch_response", "num_partitions"), [ +@pytest.mark.parametrize(("fetch_offsets", "fetch_response", "num_partitions"), [ ( - FetchRequest[0]( - -1, 100, 100, - [('foo', [(0, 0, 1000),])]), + {TopicPartition('foo', 0): 0}, FetchResponse[0]( [("foo", [(0, 0, 1000, [(0, b'xxx'),])]),]), 1, ), ( - FetchRequest[1]( - -1, 100, 100, - [('foo', [(0, 0, 1000), (1, 0, 1000),])]), + {TopicPartition('foo', 0): 0, TopicPartition('foo', 1): 0}, FetchResponse[1]( 0, [("foo", [ @@ -340,45 +397,53 @@ def test_fetched_records(fetcher, topic, mocker): 2, ), ( - FetchRequest[2]( - -1, 100, 100, - [('foo', [(0, 0, 1000),])]), + {TopicPartition('foo', 0): 0}, FetchResponse[2]( 0, [("foo", [(0, 0, 1000, [(0, b'xxx'),])]),]), 1, ), ( - FetchRequest[3]( - -1, 100, 100, 10000, - [('foo', [(0, 0, 1000),])]), + {TopicPartition('foo', 0): 0}, FetchResponse[3]( 0, [("foo", [(0, 0, 1000, [(0, b'xxx'),])]),]), 1, ), ( - FetchRequest[4]( - -1, 100, 100, 10000, 0, - [('foo', [(0, 0, 1000),])]), + {TopicPartition('foo', 0): 0}, FetchResponse[4]( 0, [("foo", [(0, 0, 1000, 0, [], [(0, b'xxx'),])]),]), 1, ), ( # This may only be used in broker-broker api calls - FetchRequest[5]( - -1, 100, 100, 10000, 0, - [('foo', [(0, 0, 1000),])]), + {TopicPartition('foo', 0): 0}, FetchResponse[5]( 0, [("foo", [(0, 0, 1000, 0, 0, [], [(0, b'xxx'),])]),]), 1, ), ]) -def test__handle_fetch_response(fetcher, fetch_request, fetch_response, num_partitions): - fetcher._handle_fetch_response(fetch_request, time.time(), fetch_response) +def test__handle_fetch_response(fetcher, fetch_offsets, fetch_response, num_partitions): + fetcher._handle_fetch_response(0, fetch_offsets, time.time(), fetch_response) assert len(fetcher._completed_fetches) == num_partitions -def test__unpack_message_set(fetcher): +@pytest.mark.parametrize(("exception", "log_level"), [ +( + Errors.Cancelled(), + logging.INFO +), +( + Errors.KafkaError(), + logging.ERROR +) +]) +def test__handle_fetch_error(fetcher, caplog, exception, log_level): + fetcher._handle_fetch_error(3, exception) + assert len(caplog.records) == 1 + assert caplog.records[0].levelname == logging.getLevelName(log_level) + + +def test__unpack_records(fetcher): fetcher.config['check_crcs'] = False tp = TopicPartition('foo', 0) messages = [ @@ -387,7 +452,7 @@ def test__unpack_message_set(fetcher): (None, b"c", None), ] memory_records = MemoryRecords(_build_record_batch(messages)) - records = list(fetcher._unpack_message_set(tp, memory_records)) + records = list(fetcher._unpack_records(tp, memory_records)) assert len(records) == 3 assert all(map(lambda x: isinstance(x, ConsumerRecord), records)) assert records[0].value == b'a' @@ -467,6 +532,7 @@ def test__parse_fetched_data__not_leader(fetcher, topic, mocker): tp, 0, 0, [NotLeaderForPartitionError.errno, -1, None], mocker.MagicMock() ) + mocker.patch.object(fetcher._client.cluster, 'request_update') partition_record = fetcher._parse_fetched_data(completed_fetch) assert partition_record is None fetcher._client.cluster.request_update.assert_called_with() @@ -479,6 +545,7 @@ def test__parse_fetched_data__unknown_tp(fetcher, topic, mocker): tp, 0, 0, [UnknownTopicOrPartitionError.errno, -1, None], mocker.MagicMock() ) + mocker.patch.object(fetcher._client.cluster, 'request_update') partition_record = fetcher._parse_fetched_data(completed_fetch) assert partition_record is None fetcher._client.cluster.request_update.assert_called_with() @@ -504,7 +571,7 @@ def test_partition_records_offset(): batch_end = 130 fetch_offset = 123 tp = TopicPartition('foo', 0) - messages = [ConsumerRecord(tp.topic, tp.partition, i, + messages = [ConsumerRecord(tp.topic, tp.partition, -1, i, None, None, 'key', 'value', [], 'checksum', 0, 0, -1) for i in range(batch_start, batch_end)] records = Fetcher.PartitionRecords(fetch_offset, None, messages) @@ -529,7 +596,7 @@ def test_partition_records_no_fetch_offset(): batch_end = 100 fetch_offset = 123 tp = TopicPartition('foo', 0) - messages = [ConsumerRecord(tp.topic, tp.partition, i, + messages = [ConsumerRecord(tp.topic, tp.partition, -1, i, None, None, 'key', 'value', None, 'checksum', 0, 0, -1) for i in range(batch_start, batch_end)] records = Fetcher.PartitionRecords(fetch_offset, None, messages) @@ -544,7 +611,7 @@ def test_partition_records_compacted_offset(): batch_end = 100 fetch_offset = 42 tp = TopicPartition('foo', 0) - messages = [ConsumerRecord(tp.topic, tp.partition, i, + messages = [ConsumerRecord(tp.topic, tp.partition, -1, i, None, None, 'key', 'value', None, 'checksum', 0, 0, -1) for i in range(batch_start, batch_end) if i != fetch_offset] records = Fetcher.PartitionRecords(fetch_offset, None, messages) diff --git a/test/test_object_conversion.py b/test/test_object_conversion.py index 9b1ff2131..a48eb0601 100644 --- a/test/test_object_conversion.py +++ b/test/test_object_conversion.py @@ -207,7 +207,7 @@ def test_with_metadata_response(): assert len(obj['topics']) == 2 assert obj['topics'][0]['error_code'] == 0 assert obj['topics'][0]['topic'] == 'testtopic1' - assert obj['topics'][0]['is_internal'] == False + assert obj['topics'][0]['is_internal'] is False assert len(obj['topics'][0]['partitions']) == 2 assert obj['topics'][0]['partitions'][0]['error_code'] == 0 assert obj['topics'][0]['partitions'][0]['partition'] == 0 @@ -224,7 +224,7 @@ def test_with_metadata_response(): assert obj['topics'][1]['error_code'] == 0 assert obj['topics'][1]['topic'] == 'other-test-topic' - assert obj['topics'][1]['is_internal'] == True + assert obj['topics'][1]['is_internal'] is True assert len(obj['topics'][1]['partitions']) == 1 assert obj['topics'][1]['partitions'][0]['error_code'] == 0 assert obj['topics'][1]['partitions'][0]['partition'] == 0 diff --git a/test/test_protocol.py b/test/test_protocol.py index 6a77e19d6..d0cc7ed0a 100644 --- a/test/test_protocol.py +++ b/test/test_protocol.py @@ -2,11 +2,9 @@ import io import struct -import pytest - from kafka.protocol.api import RequestHeader -from kafka.protocol.commit import GroupCoordinatorRequest from kafka.protocol.fetch import FetchRequest, FetchResponse +from kafka.protocol.find_coordinator import FindCoordinatorRequest from kafka.protocol.message import Message, MessageSet, PartialMessage from kafka.protocol.metadata import MetadataRequest from kafka.protocol.types import Int16, Int32, Int64, String, UnsignedVarInt32, CompactString, CompactArray, CompactBytes @@ -168,7 +166,7 @@ def test_encode_message_header(): b'client3', # ClientId ]) - req = GroupCoordinatorRequest[0]('foo') + req = FindCoordinatorRequest[0]('foo') header = RequestHeader(req, correlation_id=4, client_id='client3') assert header.encode() == expect @@ -273,7 +271,7 @@ def test_decode_fetch_response_partial(): def test_struct_unrecognized_kwargs(): try: - mr = MetadataRequest[0](topicz='foo') + _mr = MetadataRequest[0](topicz='foo') assert False, 'Structs should not allow unrecognized kwargs' except ValueError: pass @@ -331,6 +329,6 @@ def test_compact_data_structs(): assert CompactBytes.decode(io.BytesIO(b'\x00')) is None enc = CompactBytes.encode(b'') assert enc == b'\x01' - assert CompactBytes.decode(io.BytesIO(b'\x01')) is b'' + assert CompactBytes.decode(io.BytesIO(b'\x01')) == b'' enc = CompactBytes.encode(b'foo') assert CompactBytes.decode(io.BytesIO(enc)) == b'foo' diff --git a/test/test_sasl_integration.py b/test/test_sasl_integration.py index e3a4813ae..0f404da20 100644 --- a/test/test_sasl_integration.py +++ b/test/test_sasl_integration.py @@ -1,5 +1,6 @@ import logging import uuid +import time import pytest @@ -69,12 +70,17 @@ def test_client(request, sasl_kafka): client, = sasl_kafka.get_clients(1) request = MetadataRequest_v1(None) - client.send(0, request) - for _ in range(10): - result = client.poll(timeout_ms=10000) - if len(result) > 0: - break - else: + timeout_at = time.time() + 1 + while not client.is_ready(0): + client.maybe_connect(0) + client.poll(timeout_ms=100) + if time.time() > timeout_at: + raise RuntimeError("Couldn't connect to node 0") + future = client.send(0, request) + client.poll(future=future, timeout_ms=10000) + if not future.is_done: raise RuntimeError("Couldn't fetch topic response from Broker.") - result = result[0] + elif future.failed(): + raise future.exception + result = future.value assert topic_name in [t[1] for t in result.topics] diff --git a/test/test_sender.py b/test/test_sender.py index 2a68defcf..3da1a9f42 100644 --- a/test/test_sender.py +++ b/test/test_sender.py @@ -5,8 +5,8 @@ import io from kafka.client_async import KafkaClient -from kafka.cluster import ClusterMetadata from kafka.metrics import Metrics +from kafka.protocol.broker_api_versions import BROKER_API_VERSIONS from kafka.protocol.produce import ProduceRequest from kafka.producer.record_accumulator import RecordAccumulator, ProducerBatch from kafka.producer.sender import Sender @@ -15,10 +15,8 @@ @pytest.fixture -def client(mocker): - _cli = mocker.Mock(spec=KafkaClient(bootstrap_servers=(), api_version=(0, 9))) - _cli.cluster = mocker.Mock(spec=ClusterMetadata()) - return _cli +def client(): + return KafkaClient(bootstrap_servers=(), api_version=(0, 9)) @pytest.fixture @@ -32,17 +30,17 @@ def metrics(): @pytest.fixture -def sender(client, accumulator, metrics): +def sender(client, accumulator, metrics, mocker): return Sender(client, client.cluster, accumulator, metrics) @pytest.mark.parametrize(("api_version", "produce_version"), [ - ((0, 10), 2), + ((0, 10, 0), 2), ((0, 9), 1), - ((0, 8), 0) + ((0, 8, 0), 0) ]) def test_produce_request(sender, mocker, api_version, produce_version): - sender.config['api_version'] = api_version + sender._client._api_versions = BROKER_API_VERSIONS[api_version] tp = TopicPartition('foo', 0) buffer = io.BytesIO() records = MemoryRecordsBuilder( diff --git a/test/testutil.py b/test/testutil.py index ec4d70bf6..dd4e267a8 100644 --- a/test/testutil.py +++ b/test/testutil.py @@ -28,12 +28,12 @@ def env_kafka_version(): def assert_message_count(messages, num_messages): """Check that we received the expected number of messages with no duplicates.""" # Make sure we got them all - assert len(messages) == num_messages + assert len(messages) == num_messages, 'Expected %d messages, got %d' % (num_messages, len(messages)) # Make sure there are no duplicates # Note: Currently duplicates are identified only using key/value. Other attributes like topic, partition, headers, # timestamp, etc are ignored... this could be changed if necessary, but will be more tolerant of dupes. unique_messages = {(m.key, m.value) for m in messages} - assert len(unique_messages) == num_messages + assert len(unique_messages) == num_messages, 'Expected %d unique messages, got %d' % (num_messages, len(unique_messages)) class Timer(object): diff --git a/tox.ini b/tox.ini deleted file mode 100644 index d9b1e36d4..000000000 --- a/tox.ini +++ /dev/null @@ -1,50 +0,0 @@ -[tox] -envlist = py{38,39,310,311,312,py}, docs - -[pytest] -testpaths = kafka test -addopts = --durations=10 -log_format = %(created)f %(filename)-23s %(threadName)s %(message)s - -[gh-actions] -python = - 3.8: py38 - 3.9: py39 - 3.10: py310 - 3.11: py311 - 3.12: py312 - pypy-3.9: pypy - -[testenv] -deps = - pytest - pytest-cov - pylint - pytest-pylint - pytest-mock - mock - python-snappy - zstandard - lz4 - xxhash - crc32c -commands = - pytest {posargs:--pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF --cov=kafka --cov-config=.covrc} -setenv = - CRC32C_SW_MODE = auto - PROJECT_ROOT = {toxinidir} -passenv = KAFKA_VERSION - - -[testenv:pypy] -# pylint is super slow on pypy... -commands = pytest {posargs:--cov=kafka --cov-config=.covrc} - -[testenv:docs] -deps = - sphinx_rtd_theme - sphinx - -commands = - sphinx-apidoc -o docs/apidoc/ kafka/ - sphinx-build -b html docs/ docs/_build diff --git a/travis_java_install.sh b/travis_java_install.sh deleted file mode 100755 index f662ce274..000000000 --- a/travis_java_install.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# borrowed from: https://github.com/mansenfranzen/pywrangler/blob/master/tests/travis_java_install.sh - -# Kafka requires Java 8 in order to work properly. However, TravisCI's Ubuntu -# 16.04 ships with Java 11 and Java can't be set with `jdk` when python is -# selected as language. Ubuntu 14.04 does not work due to missing python 3.7 -# support on TravisCI which does have Java 8 as default. - -# show current JAVA_HOME and java version -echo "Current JAVA_HOME: $JAVA_HOME" -echo "Current java -version:" -which java -java -version - -echo "Updating JAVA_HOME" -# change JAVA_HOME to Java 8 -export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 - -echo "Updating PATH" -export PATH=${PATH/\/usr\/local\/lib\/jvm\/openjdk11\/bin/$JAVA_HOME\/bin} - -echo "New java -version" -which java -java -version