diff --git a/.gitignore b/.gitignore index 72f931b..b01979f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,9 @@ wheelhouse vcpkg_installed/ *.pyd *.lib + + +lib_pulsar.so +tests/test.log +.tests-container-id.txt + diff --git a/CMakeLists.txt b/CMakeLists.txt index 98c1f62..493bfb0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,9 +80,7 @@ endif() set(PYTHON_WRAPPER_LIBS ${PULSAR_LIBRARY} ) -if (MSVC) - set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module) -endif () +set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module) message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}") diff --git a/RELEASE.md b/RELEASE.md index f323414..e9f9ea9 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -219,29 +219,37 @@ Then, create a PR in [`pulsar-site`](https://github.com/apache/pulsar-site) repo For minor releases, skip this section. For major releases, you should generate the HTML files into the [`pulsar-site`](https://github.com/apache/pulsar-site) repo: ```bash +# Use the first two version numbers, e.g. export VERSION=3.2 +VERSION=X.Y + +# You need to install the wheel to have the _pulsar.so installed +# It's better to run the following commands in an empty directory +python3 -m pip install pulsar-client==$VERSION.0 --force-reinstall +C_MODULE_PATH=$(python3 -c 'import _pulsar, os; print(_pulsar.__file__)') + git clone git@github.com:apache/pulsar-client-python.git cd pulsar-client-python -git checkout vX.Y.0 -# It's better to replace this URL with the URL of your own fork +git checkout v$VERSION.0 +# You can skip this step if you already have the `pulsar-site` repository in your local env. +# In this case, you only need to modify the `--html-output` parameter in the following command. git clone git@github.com:apache/pulsar-site.git sudo python3 -m pip install pydoctor -cp $(python3 -c 'import _pulsar, os; print(_pulsar.__file__)') ./_pulsar.so pydoctor --make-html \ - --html-viewsource-base=https://github.com/apache/pulsar-client-python/tree/vX.Y.0 \ + --html-viewsource-base=https://github.com/apache/pulsar-client-python/tree/v$VERSION.0 \ --docformat=numpy --theme=readthedocs \ --intersphinx=https://docs.python.org/3/objects.inv \ - --html-output=./pulsar-site/site2/website-next/static/api/python/X.Y.x \ + --html-output=./pulsar-site/static/api/python/$VERSION.x \ --introspect-c-modules \ - ./_pulsar.so \ + $C_MODULE_PATH \ pulsar cd pulsar-site -git checkout -b py-docs-X.Y +git checkout -b py-docs-$VERSION git add . -git commit -m "Generate Python client X.Y.0 doc" -git push origin py-docs-X.Y +git commit -m "Generate Python client $VERSION.0 doc" +git push origin py-docs-$VERSION ``` -Then open a PR like: https://github.com/apache/pulsar-site/pull/342 +Then open a PR like: https://github.com/apache/pulsar-site/pull/600 ## Announce the release diff --git a/dependencies.yaml b/dependencies.yaml index e8f0362..edd8f17 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -18,7 +18,7 @@ # cmake: 3.24.2 -pulsar-cpp: 3.2.0 +pulsar-cpp: 3.3.0 pybind11: 2.10.1 boost: 1.80.0 protobuf: 3.20.0 diff --git a/pkg/mac/build-dependencies.sh b/pkg/mac/build-dependencies.sh index 84f48dc..e317751 100755 --- a/pkg/mac/build-dependencies.sh +++ b/pkg/mac/build-dependencies.sh @@ -150,10 +150,12 @@ if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then echo "Building Protobuf" download_dependency $ROOT_DIR/dependencies.yaml protobuf pushd protobuf-${PROTOBUF_VERSION} - CXXFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ - ./configure --prefix=$PREFIX - make -j16 - make install + # Install by CMake so that the dependency can be found with CMake config mode + pushd cmake/ + cmake -B build -DCMAKE_CXX_FLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + -DCMAKE_INSTALL_PREFIX=$PREFIX + cmake --build build -j16 --target install + popd touch .done popd else @@ -194,7 +196,9 @@ if [ ! -f curl-${CURL_VERSION}/.done ]; then CURL_VERSION_=${CURL_VERSION//./_} download_dependency $ROOT_DIR/dependencies.yaml curl pushd curl-${CURL_VERSION} - CFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ + # Force the compiler to find the OpenSSL headers instead of the headers in the system path like /usr/local/include/openssl. + cp -rf $PREFIX/include/openssl include/ + CFLAGS="-I$PREFIX/include -fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ ./configure --with-ssl=$PREFIX \ --without-nghttp2 \ --without-libidn2 \ diff --git a/pkg/mac/build-pulsar-cpp.sh b/pkg/mac/build-pulsar-cpp.sh index a6849e2..51f1ef3 100755 --- a/pkg/mac/build-pulsar-cpp.sh +++ b/pkg/mac/build-pulsar-cpp.sh @@ -45,12 +45,14 @@ if [ ! -f apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}/.done ]; then ARCHS='arm64;x86_64' cmake . \ + -DCMAKE_CXX_STANDARD=11 \ -DCMAKE_OSX_ARCHITECTURES=${ARCHS} \ -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_PREFIX_PATH=${DEPS_PREFIX} \ -DCMAKE_CXX_FLAGS=-I${DEPS_PREFIX}/include \ + -DOPENSSL_ROOT_DIR=${DEPS_PREFIX} \ -DLINK_STATIC=OFF \ -DBUILD_TESTS=OFF \ -DBUILD_WIRESHARK=OFF \ diff --git a/pulsar/__about__.py b/pulsar/__about__.py index 1c296a7..3726341 100644 --- a/pulsar/__about__.py +++ b/pulsar/__about__.py @@ -16,4 +16,4 @@ # specific language governing permissions and limitations # under the License. # -__version__='3.2.0a1' +__version__='3.3.0' diff --git a/pulsar/__init__.py b/pulsar/__init__.py index dbf3d82..8e0907e 100644 --- a/pulsar/__init__.py +++ b/pulsar/__init__.py @@ -43,10 +43,13 @@ """ import logging +from typing import List, Tuple, Optional + import _pulsar from _pulsar import Result, CompressionType, ConsumerType, InitialPosition, PartitionsRoutingMode, BatchingType, \ - LoggerLevel, BatchReceivePolicy # noqa: F401 + LoggerLevel, BatchReceivePolicy, KeySharedPolicy, KeySharedMode, ProducerAccessMode, RegexSubscriptionMode, \ + DeadLetterPolicyBuilder # noqa: F401 from pulsar.__about__ import __version__ @@ -372,6 +375,65 @@ def __init__(self, username=None, password=None, method='basic', auth_params_str _check_type(str, method, 'method') self.auth = _pulsar.AuthenticationBasic.create(username, password, method) +class ConsumerDeadLetterPolicy: + """ + Configuration for the "dead letter queue" feature in consumer. + """ + def __init__(self, + max_redeliver_count: int, + dead_letter_topic: str = None, + initial_subscription_name: str = None): + """ + Wrapper DeadLetterPolicy. + + Parameters + ---------- + max_redeliver_count: Maximum number of times that a message is redelivered before being sent to the dead letter queue. + - The maxRedeliverCount must be greater than 0. + dead_letter_topic: Name of the dead topic where the failing messages are sent. + The default value is: sourceTopicName + "-" + subscriptionName + "-DLQ" + initial_subscription_name: Name of the initial subscription name of the dead letter topic. + If this field is not set, the initial subscription for the dead letter topic is not created. + If this field is set but the broker's `allowAutoSubscriptionCreation` is disabled, the DLQ producer + fails to be created. + """ + builder = DeadLetterPolicyBuilder() + if max_redeliver_count is None or max_redeliver_count < 1: + raise ValueError("max_redeliver_count must be greater than 0") + builder.maxRedeliverCount(max_redeliver_count) + if dead_letter_topic is not None: + builder.deadLetterTopic(dead_letter_topic) + if initial_subscription_name is not None: + builder.initialSubscriptionName(initial_subscription_name) + self._policy = builder.build() + + @property + def dead_letter_topic(self) -> str: + """ + Return the dead letter topic for dead letter policy. + """ + return self._policy.getDeadLetterTopic() + + @property + def max_redeliver_count(self) -> int: + """ + Return the max redeliver count for dead letter policy. + """ + return self._policy.getMaxRedeliverCount() + + @property + def initial_subscription_name(self) -> str: + """ + Return the initial subscription name for dead letter policy. + """ + return self._policy.getInitialSubscriptionName() + + def policy(self): + """ + Returns the actual one DeadLetterPolicy. + """ + return self._policy + class Client: """ The Pulsar client. A single client instance can be used to create producers @@ -424,7 +486,8 @@ def __init__(self, service_url, Number of concurrent lookup-requests allowed on each broker connection to prevent overload on the broker. log_conf_file_path: str, optional - Initialize log4cxx from a configuration file. + This parameter is deprecated and makes no effect. It's retained only for compatibility. + Use `logger` to customize a logger. use_tls: bool, default=False Configure whether to use TLS encryption on the connection. This setting is deprecated. TLS will be automatically enabled if the ``serviceUrl`` is set to ``pulsar+ssl://`` or ``https://`` @@ -466,8 +529,6 @@ def __init__(self, service_url, conf.io_threads(io_threads) conf.message_listener_threads(message_listener_threads) conf.concurrent_lookup_requests(concurrent_lookup_requests) - if log_conf_file_path: - conf.log_conf_file_path(log_conf_file_path) if isinstance(logger, logging.Logger): conf.set_logger(self._prepare_logger(logger)) @@ -521,7 +582,8 @@ def create_producer(self, topic, properties=None, batching_type=BatchingType.Default, encryption_key=None, - crypto_key_reader=None + crypto_key_reader=None, + access_mode=ProducerAccessMode.Shared, ): """ Create a new producer on a given topic. @@ -612,6 +674,17 @@ def create_producer(self, topic, crypto_key_reader: CryptoKeyReader, optional Symmetric encryption class implementation, configuring public key encryption messages for the producer and private key decryption messages for the consumer + access_mode: ProducerAccessMode, optional + Set the type of access mode that the producer requires on the topic. + + Supported modes: + + * Shared: By default multiple producers can publish on a topic. + * Exclusive: Require exclusive access for producer. + Fail immediately if there's already a producer connected. + * WaitForExclusive: Producer creation is pending until it can acquire exclusive access. + * ExclusiveWithFencing: Acquire exclusive access for the producer. + Any existing producer will be removed and invalidated immediately. """ _check_type(str, topic, 'topic') _check_type_or_none(str, producer_name, 'producer_name') @@ -632,6 +705,7 @@ def create_producer(self, topic, _check_type_or_none(str, encryption_key, 'encryption_key') _check_type_or_none(CryptoKeyReader, crypto_key_reader, 'crypto_key_reader') _check_type(bool, lazy_start_partitioned_producers, 'lazy_start_partitioned_producers') + _check_type(ProducerAccessMode, access_mode, 'access_mode') conf = _pulsar.ProducerConfiguration() conf.send_timeout_millis(send_timeout_millis) @@ -647,6 +721,7 @@ def create_producer(self, topic, conf.batching_type(batching_type) conf.chunking_enabled(chunking_enabled) conf.lazy_start_partitioned_producers(lazy_start_partitioned_producers) + conf.access_mode(access_mode) if producer_name: conf.producer_name(producer_name) if initial_sequence_id: @@ -689,7 +764,11 @@ def subscribe(self, topic, subscription_name, max_pending_chunked_message=10, auto_ack_oldest_chunked_message_on_queue_full=False, start_message_id_inclusive=False, - batch_receive_policy=None + batch_receive_policy=None, + key_shared_policy=None, + batch_index_ack_enabled=False, + regex_subscription_mode=RegexSubscriptionMode.PersistentOnly, + dead_letter_policy: ConsumerDeadLetterPolicy = None, ): """ Subscribe to the given topic and subscription combination. @@ -774,6 +853,25 @@ def my_listener(consumer, message): Set the consumer to include the given position of any reset operation like Consumer::seek. batch_receive_policy: class ConsumerBatchReceivePolicy Set the batch collection policy for batch receiving. + key_shared_policy: class ConsumerKeySharedPolicy + Set the key shared policy for use when the ConsumerType is KeyShared. + batch_index_ack_enabled: Enable the batch index acknowledgement. + It should be noted that this option can only work when the broker side also enables the batch index + acknowledgement. See the `acknowledgmentAtBatchIndexLevelEnabled` config in `broker.conf`. + regex_subscription_mode: RegexSubscriptionMode, optional + Set the regex subscription mode for use when the topic is a regex pattern. + + Supported modes: + + * PersistentOnly: By default only subscribe to persistent topics. + * NonPersistentOnly: Only subscribe to non-persistent topics. + * AllTopics: Subscribe to both persistent and non-persistent topics. + dead_letter_policy: class ConsumerDeadLetterPolicy + Set dead letter policy for consumer. + By default, some messages are redelivered many times, even to the extent that they can never be + stopped. By using the dead letter mechanism, messages have the max redelivery count, when they're + exceeding the maximum number of redeliveries. Messages are sent to dead letter topics and acknowledged + automatically. """ _check_type(str, subscription_name, 'subscription_name') _check_type(ConsumerType, consumer_type, 'consumer_type') @@ -794,9 +892,13 @@ def my_listener(consumer, message): _check_type(bool, auto_ack_oldest_chunked_message_on_queue_full, 'auto_ack_oldest_chunked_message_on_queue_full') _check_type(bool, start_message_id_inclusive, 'start_message_id_inclusive') _check_type_or_none(ConsumerBatchReceivePolicy, batch_receive_policy, 'batch_receive_policy') + _check_type_or_none(ConsumerKeySharedPolicy, key_shared_policy, 'key_shared_policy') + _check_type(bool, batch_index_ack_enabled, 'batch_index_ack_enabled') + _check_type(RegexSubscriptionMode, regex_subscription_mode, 'regex_subscription_mode') conf = _pulsar.ConsumerConfiguration() conf.consumer_type(consumer_type) + conf.regex_subscription_mode(regex_subscription_mode) conf.read_compacted(is_read_compacted) if message_listener: conf.message_listener(_listener_wrapper(message_listener, schema)) @@ -826,6 +928,12 @@ def my_listener(consumer, message): if batch_receive_policy: conf.batch_receive_policy(batch_receive_policy.policy()) + if key_shared_policy: + conf.key_shared_policy(key_shared_policy.policy()) + conf.batch_index_ack_enabled(batch_index_ack_enabled) + if dead_letter_policy: + conf.dead_letter_policy(dead_letter_policy.policy()) + c = Consumer() if isinstance(topic, str): # Single topic @@ -1448,6 +1556,73 @@ def policy(self): """ return self._policy +class ConsumerKeySharedPolicy: + """ + Consumer key shared policy is used to configure the consumer behaviour when the ConsumerType is KeyShared. + """ + def __init__( + self, + key_shared_mode: KeySharedMode = KeySharedMode.AutoSplit, + allow_out_of_order_delivery: bool = False, + sticky_ranges: Optional[List[Tuple[int, int]]] = None, + ): + """ + Wrapper KeySharedPolicy. + + Parameters + ---------- + + key_shared_mode: KeySharedMode, optional + Set the key shared mode. eg: KeySharedMode.Sticky or KeysharedMode.AutoSplit + + allow_out_of_order_delivery: bool, optional + Set whether to allow for out of order delivery + If it is enabled, it relaxes the ordering requirement and allows the broker to send out-of-order + messages in case of failures. This makes it faster for new consumers to join without being stalled by + an existing slow consumer. + + If this is True, a single consumer still receives all keys, but they may come in different orders. + + sticky_ranges: List[Tuple[int, int]], optional + Set the ranges used with sticky mode. The integers can be from 0 to 2^16 (0 <= val < 65,536) + """ + if key_shared_mode == KeySharedMode.Sticky and sticky_ranges is None: + raise ValueError("When using key_shared_mode = KeySharedMode.Sticky you must also provide sticky_ranges") + + self._policy = KeySharedPolicy() + self._policy.set_key_shared_mode(key_shared_mode) + self._policy.set_allow_out_of_order_delivery(allow_out_of_order_delivery) + + if sticky_ranges is not None: + self._policy.set_sticky_ranges(sticky_ranges) + + @property + def key_shared_mode(self) -> KeySharedMode: + """ + Returns the key shared mode + """ + return self._policy.get_key_shared_mode() + + @property + def allow_out_of_order_delivery(self) -> bool: + """ + Returns whether out of order delivery is enabled + """ + return self._policy.is_allow_out_of_order_delivery() + + @property + def sticky_ranges(self) -> List[Tuple[int, int]]: + """ + Returns the actual sticky ranges + """ + return self._policy.get_sticky_ranges() + + def policy(self): + """ + Returns the actual KeySharedPolicy. + """ + return self._policy + class Reader: """ Pulsar topic reader. diff --git a/pulsar/schema/schema_avro.py b/pulsar/schema/schema_avro.py index 70fda98..480afe9 100644 --- a/pulsar/schema/schema_avro.py +++ b/pulsar/schema/schema_avro.py @@ -91,7 +91,7 @@ def decode_message(self, msg: _pulsar.Message): writer_schema = self._get_writer_schema(topic, version) return self._decode_bytes(msg.data(), writer_schema) except Exception as e: - self._logger.error('Failed to get schema info of {topic} version {version}: {e}') + self._logger.error(f'Failed to get schema info of {topic} version {version}: {e}') return self._decode_bytes(msg.data(), self._schema) def _get_writer_schema(self, topic: str, version: int) -> 'dict': diff --git a/src/config.cc b/src/config.cc index 71795dd..ac643b7 100644 --- a/src/config.cc +++ b/src/config.cc @@ -21,8 +21,11 @@ #include #include #include +#include +#include #include #include +#include #include namespace py = pybind11; @@ -121,6 +124,15 @@ static ClientConfiguration& ClientConfiguration_setFileLogger(ClientConfiguratio void export_config(py::module_& m) { using namespace py; + class_>(m, "KeySharedPolicy") + .def(init<>()) + .def("set_key_shared_mode", &KeySharedPolicy::setKeySharedMode, return_value_policy::reference) + .def("get_key_shared_mode", &KeySharedPolicy::getKeySharedMode) + .def("set_allow_out_of_order_delivery", &KeySharedPolicy::setAllowOutOfOrderDelivery, return_value_policy::reference) + .def("is_allow_out_of_order_delivery", &KeySharedPolicy::isAllowOutOfOrderDelivery) + .def("set_sticky_ranges", static_cast(&KeySharedPolicy::setStickyRanges), return_value_policy::reference) + .def("get_sticky_ranges", &KeySharedPolicy::getStickyRanges); + class_>(m, "AbstractCryptoKeyReader") .def("getPublicKey", &CryptoKeyReader::getPublicKey) .def("getPrivateKey", &CryptoKeyReader::getPrivateKey); @@ -145,8 +157,6 @@ void export_config(py::module_& m) { .def("concurrent_lookup_requests", &ClientConfiguration::getConcurrentLookupRequest) .def("concurrent_lookup_requests", &ClientConfiguration::setConcurrentLookupRequest, return_value_policy::reference) - .def("log_conf_file_path", &ClientConfiguration::getLogConfFilePath, return_value_policy::copy) - .def("log_conf_file_path", &ClientConfiguration::setLogConfFilePath, return_value_policy::reference) .def("use_tls", &ClientConfiguration::isUseTls) .def("use_tls", &ClientConfiguration::setUseTls, return_value_policy::reference) .def("tls_trust_certs_file_path", &ClientConfiguration::getTlsTrustCertsFilePath, @@ -212,7 +222,9 @@ void export_config(py::module_& m) { .def("batching_type", &ProducerConfiguration::setBatchingType, return_value_policy::reference) .def("batching_type", &ProducerConfiguration::getBatchingType) .def("encryption_key", &ProducerConfiguration::addEncryptionKey, return_value_policy::reference) - .def("crypto_key_reader", &ProducerConfiguration::setCryptoKeyReader, return_value_policy::reference); + .def("crypto_key_reader", &ProducerConfiguration::setCryptoKeyReader, return_value_policy::reference) + .def("access_mode", &ProducerConfiguration::setAccessMode, return_value_policy::reference) + .def("access_mode", &ProducerConfiguration::getAccessMode, return_value_policy::copy); class_(m, "BatchReceivePolicy") .def(init()) @@ -220,8 +232,24 @@ void export_config(py::module_& m) { .def("getMaxNumMessages", &BatchReceivePolicy::getMaxNumMessages) .def("getMaxNumBytes", &BatchReceivePolicy::getMaxNumBytes); + class_(m, "DeadLetterPolicy") + .def(init<>()) + .def("getDeadLetterTopic", &DeadLetterPolicy::getDeadLetterTopic) + .def("getMaxRedeliverCount", &DeadLetterPolicy::getMaxRedeliverCount) + .def("getInitialSubscriptionName", &DeadLetterPolicy::getInitialSubscriptionName); + + class_(m, "DeadLetterPolicyBuilder") + .def(init<>()) + .def("deadLetterTopic", &DeadLetterPolicyBuilder::deadLetterTopic, return_value_policy::reference) + .def("maxRedeliverCount", &DeadLetterPolicyBuilder::maxRedeliverCount, return_value_policy::reference) + .def("initialSubscriptionName", &DeadLetterPolicyBuilder::initialSubscriptionName, return_value_policy::reference) + .def("build", &DeadLetterPolicyBuilder::build, return_value_policy::reference) + .def("build", &DeadLetterPolicyBuilder::build, return_value_policy::reference); + class_>(m, "ConsumerConfiguration") .def(init<>()) + .def("key_shared_policy", &ConsumerConfiguration::getKeySharedPolicy) + .def("key_shared_policy", &ConsumerConfiguration::setKeySharedPolicy, return_value_policy::reference) .def("consumer_type", &ConsumerConfiguration::getConsumerType) .def("consumer_type", &ConsumerConfiguration::setConsumerType, return_value_policy::reference) .def("schema", &ConsumerConfiguration::getSchema, return_value_policy::copy) @@ -252,6 +280,8 @@ void export_config(py::module_& m) { .def("property", &ConsumerConfiguration::setProperty, return_value_policy::reference) .def("subscription_initial_position", &ConsumerConfiguration::getSubscriptionInitialPosition) .def("subscription_initial_position", &ConsumerConfiguration::setSubscriptionInitialPosition) + .def("regex_subscription_mode", &ConsumerConfiguration::setRegexSubscriptionMode) + .def("regex_subscription_mode", &ConsumerConfiguration::getRegexSubscriptionMode, return_value_policy::reference) .def("crypto_key_reader", &ConsumerConfiguration::setCryptoKeyReader, return_value_policy::reference) .def("replicate_subscription_state_enabled", &ConsumerConfiguration::setReplicateSubscriptionStateEnabled) @@ -267,7 +297,12 @@ void export_config(py::module_& m) { return_value_policy::reference) .def("start_message_id_inclusive", &ConsumerConfiguration::isStartMessageIdInclusive) .def("start_message_id_inclusive", &ConsumerConfiguration::setStartMessageIdInclusive, - return_value_policy::reference); + return_value_policy::reference) + .def("batch_index_ack_enabled", &ConsumerConfiguration::isBatchIndexAckEnabled) + .def("batch_index_ack_enabled", &ConsumerConfiguration::setBatchIndexAckEnabled, + return_value_policy::reference) + .def("dead_letter_policy", &ConsumerConfiguration::setDeadLetterPolicy) + .def("dead_letter_policy", &ConsumerConfiguration::getDeadLetterPolicy, return_value_policy::copy); class_>(m, "ReaderConfiguration") .def(init<>()) diff --git a/src/enums.cc b/src/enums.cc index f61011f..198edfa 100644 --- a/src/enums.cc +++ b/src/enums.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include using namespace pulsar; @@ -28,6 +29,10 @@ namespace py = pybind11; void export_enums(py::module_& m) { using namespace py; + enum_(m, "KeySharedMode") + .value("AutoSplit", AUTO_SPLIT) + .value("Sticky", STICKY); + enum_(m, "PartitionsRoutingMode") .value("UseSinglePartition", ProducerConfiguration::UseSinglePartition) .value("RoundRobinDistribution", ProducerConfiguration::RoundRobinDistribution) @@ -115,10 +120,21 @@ void export_enums(py::module_& m) { .value("Latest", InitialPositionLatest) .value("Earliest", InitialPositionEarliest); + enum_(m, "RegexSubscriptionMode", "Regex subscription mode") + .value("PersistentOnly", PersistentOnly) + .value("NonPersistentOnly", NonPersistentOnly) + .value("AllTopics", AllTopics); + enum_(m, "BatchingType", "Supported batching types") .value("Default", ProducerConfiguration::DefaultBatching) .value("KeyBased", ProducerConfiguration::KeyBasedBatching); + enum_(m, "ProducerAccessMode", "Producer Access Mode") + .value("Shared", ProducerConfiguration::ProducerAccessMode::Shared) + .value("Exclusive", ProducerConfiguration::ProducerAccessMode::Exclusive) + .value("WaitForExclusive", ProducerConfiguration::ProducerAccessMode::WaitForExclusive) + .value("ExclusiveWithFencing", ProducerConfiguration::ProducerAccessMode::ExclusiveWithFencing); + enum_(m, "LoggerLevel") .value("Debug", Logger::LEVEL_DEBUG) .value("Info", Logger::LEVEL_INFO) diff --git a/tests/pulsar_test.py b/tests/pulsar_test.py index eeb2a6a..ae6aa3b 100755 --- a/tests/pulsar_test.py +++ b/tests/pulsar_test.py @@ -24,6 +24,7 @@ from unittest import TestCase, main import time import os +import re import pulsar import uuid from datetime import timedelta @@ -32,6 +33,8 @@ MessageId, CompressionType, ConsumerType, + KeySharedMode, + ConsumerKeySharedPolicy, PartitionsRoutingMode, AuthenticationBasic, AuthenticationTLS, @@ -40,10 +43,12 @@ InitialPosition, CryptoKeyReader, ConsumerBatchReceivePolicy, + ProducerAccessMode, + ConsumerDeadLetterPolicy, ) from pulsar.schema import JsonSchema, Record, Integer -from _pulsar import ProducerConfiguration, ConsumerConfiguration +from _pulsar import ProducerConfiguration, ConsumerConfiguration, RegexSubscriptionMode from schema_test import * @@ -164,6 +169,63 @@ def test_producer_send(self): self.assertEqual(msg_id, msg.message_id()) client.close() + def test_producer_access_mode_exclusive(self): + client = Client(self.serviceUrl) + topic_name = "test-access-mode-exclusive" + client.create_producer(topic_name, producer_name="p1", access_mode=ProducerAccessMode.Exclusive) + with self.assertRaises(pulsar.ProducerFenced): + client.create_producer(topic_name, producer_name="p2", access_mode=ProducerAccessMode.Exclusive) + client.close() + + def test_producer_access_mode_wait_exclusive(self): + client = Client(self.serviceUrl) + topic_name = "test_producer_access_mode_wait_exclusive" + producer1 = client.create_producer( + topic=topic_name, + producer_name='p-1', + access_mode=ProducerAccessMode.Exclusive + ) + assert producer1.producer_name() == 'p-1' + + # when p1 close, p2 success created. + producer1.close() + producer2 = client.create_producer( + topic=topic_name, + producer_name='p-2', + access_mode=ProducerAccessMode.WaitForExclusive + ) + assert producer2.producer_name() == 'p-2' + + producer2.close() + client.close() + + def test_producer_access_mode_exclusive_with_fencing(self): + client = Client(self.serviceUrl) + topic_name = 'test_producer_access_mode_exclusive_with_fencing' + + producer1 = client.create_producer( + topic=topic_name, + producer_name='p-1', + access_mode=ProducerAccessMode.Exclusive + ) + assert producer1.producer_name() == 'p-1' + + producer2 = client.create_producer( + topic=topic_name, + producer_name='p-2', + access_mode=ProducerAccessMode.ExclusiveWithFencing + ) + assert producer2.producer_name() == 'p-2' + + # producer1 will be fenced. + with self.assertRaises((pulsar.ProducerFenced, pulsar.AlreadyClosed)): + producer1.send('test-msg'.encode('utf-8')) + # sleep 200ms to make sure producer1 is close done. + time.sleep(0.2) + + producer2.close() + client.close() + def test_producer_is_connected(self): client = Client(self.serviceUrl) topic = "test_producer_is_connected" @@ -570,6 +632,33 @@ def test_reader_on_specific_message_with_batches(self): reader2.close() client.close() + def test_reader_on_partitioned_topic(self): + num_of_msgs = 100 + topic_name = "public/default/my-python-topic-test_reader_on_partitioned_topic" + url1 = self.adminUrl + "/admin/v2/persistent/" + topic_name + "/partitions" + doHttpPut(url1, "4") + + client = Client(self.serviceUrl) + producer = client.create_producer(topic_name) + + send_array = [] + for i in range(num_of_msgs): + data = b"hello-%d" % i + producer.send(data) + send_array.append(data) + + reader = client.create_reader(topic_name, MessageId.earliest) + + read_array = [] + for i in range(num_of_msgs): + msg = reader.read_next(TM) + self.assertTrue(msg) + read_array.append(msg.data()) + + self.assertListEqual(sorted(send_array), sorted(read_array)) + reader.close() + client.close() + def test_reader_is_connected(self): client = Client(self.serviceUrl) topic = "test_reader_is_connected" @@ -686,7 +775,6 @@ def test_client_argument_errors(self): self._check_value_error(lambda: Client(self.serviceUrl, io_threads="test")) self._check_value_error(lambda: Client(self.serviceUrl, message_listener_threads="test")) self._check_value_error(lambda: Client(self.serviceUrl, concurrent_lookup_requests="test")) - self._check_value_error(lambda: Client(self.serviceUrl, log_conf_file_path=5)) self._check_value_error(lambda: Client(self.serviceUrl, use_tls="test")) self._check_value_error(lambda: Client(self.serviceUrl, tls_trust_certs_file_path=5)) self._check_value_error(lambda: Client(self.serviceUrl, tls_allow_insecure_connection="test")) @@ -1014,7 +1102,6 @@ def test_topics_consumer(self): client.close() def test_topics_pattern_consumer(self): - import re client = Client(self.serviceUrl) @@ -1437,6 +1524,134 @@ def send_callback(res, msg): producer.flush() client.close() + def test_keyshare_policy(self): + with self.assertRaises(ValueError): + # Raise error because sticky ranges are not provided. + pulsar.ConsumerKeySharedPolicy( + key_shared_mode=pulsar.KeySharedMode.Sticky, + allow_out_of_order_delivery=False, + ) + + expected_key_shared_mode = pulsar.KeySharedMode.Sticky + expected_allow_out_of_order_delivery = True + expected_sticky_ranges = [(0, 100), (101,200)] + consumer_key_shared_policy = pulsar.ConsumerKeySharedPolicy( + key_shared_mode=expected_key_shared_mode, + allow_out_of_order_delivery=expected_allow_out_of_order_delivery, + sticky_ranges=expected_sticky_ranges + ) + + self.assertEqual(consumer_key_shared_policy.key_shared_mode, expected_key_shared_mode) + self.assertEqual(consumer_key_shared_policy.allow_out_of_order_delivery, expected_allow_out_of_order_delivery) + self.assertEqual(consumer_key_shared_policy.sticky_ranges, expected_sticky_ranges) + + def test_keyshared_invalid_sticky_ranges(self): + client = Client(self.serviceUrl) + topic = "my-python-topic-keyshare-invalid-" + str(time.time()) + with self.assertRaises(ValueError): + consumer_key_shared_policy = pulsar.ConsumerKeySharedPolicy( + key_shared_mode=pulsar.KeySharedMode.Sticky, + allow_out_of_order_delivery=False, + sticky_ranges=[(0,65536)] + ) + client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, + start_message_id_inclusive=True, + key_shared_policy=consumer_key_shared_policy) + + with self.assertRaises(ValueError): + consumer_key_shared_policy = pulsar.ConsumerKeySharedPolicy( + key_shared_mode=pulsar.KeySharedMode.Sticky, + allow_out_of_order_delivery=False, + sticky_ranges=[(0, 100), (50, 150)] + ) + client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, + start_message_id_inclusive=True, + key_shared_policy=consumer_key_shared_policy) + + def test_keyshared_autosplit(self): + client = Client(self.serviceUrl) + topic = "my-python-topic-keyshare-autosplit-" + str(time.time()) + consumer_key_shared_policy = pulsar.ConsumerKeySharedPolicy( + key_shared_mode=pulsar.KeySharedMode.AutoSplit, + allow_out_of_order_delivery=True, + ) + consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, consumer_name = 'con-1', + start_message_id_inclusive=True, key_shared_policy=consumer_key_shared_policy) + consumer2 = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, consumer_name = 'con-2', + start_message_id_inclusive=True, key_shared_policy=consumer_key_shared_policy) + producer = client.create_producer(topic) + + for i in range(10): + if i > 0: + time.sleep(0.02) + producer.send(b"hello-%d" % i) + + msgs = [] + while True: + try: + msg = consumer.receive(100) + except pulsar.Timeout: + break + msgs.append(msg) + consumer.acknowledge(msg) + + while True: + try: + msg = consumer2.receive(100) + except pulsar.Timeout: + break + msgs.append(msg) + consumer2.acknowledge(msg) + + self.assertEqual(len(msgs), 10) + client.close() + + def test_sticky_autosplit(self): + client = Client(self.serviceUrl) + topic = "my-python-topic-keyshare-sticky-" + str(time.time()) + consumer_key_shared_policy = pulsar.ConsumerKeySharedPolicy( + key_shared_mode=pulsar.KeySharedMode.Sticky, + allow_out_of_order_delivery=True, + sticky_ranges=[(0,30000)], + ) + + consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, consumer_name='con-1', + start_message_id_inclusive=True, key_shared_policy=consumer_key_shared_policy) + + consumer2_key_shared_policy = pulsar.ConsumerKeySharedPolicy( + key_shared_mode=pulsar.KeySharedMode.Sticky, + allow_out_of_order_delivery=True, + sticky_ranges=[(30001, 65535)], + ) + consumer2 = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, consumer_name='con-2', + start_message_id_inclusive=True, key_shared_policy=consumer2_key_shared_policy) + producer = client.create_producer(topic) + + for i in range(10): + if i > 0: + time.sleep(0.02) + producer.send(b"hello-%d" % i) + + msgs = [] + while True: + try: + msg = consumer.receive(100) + except pulsar.Timeout: + break + msgs.append(msg) + consumer.acknowledge(msg) + + while True: + try: + msg = consumer2.receive(100) + except pulsar.Timeout: + break + msgs.append(msg) + consumer2.acknowledge(msg) + + self.assertEqual(len(msgs), 10) + client.close() + def test_acknowledge_failed(self): client = Client(self.serviceUrl) topic = 'test_acknowledge_failed' @@ -1460,6 +1675,163 @@ def test_acknowledge_failed(self): consumer.acknowledge(msg_id) client.close() + def test_batch_index_ack(self): + topic_name = 'test-batch-index-ack-3' + client = pulsar.Client('pulsar://localhost:6650') + producer = client.create_producer(topic_name, + batching_enabled=True, + batching_max_messages=100, + batching_max_publish_delay_ms=10000) + consumer = client.subscribe(topic_name, + subscription_name='test-batch-index-ack', + batch_index_ack_enabled=True) + + # Make sure send 0~5 is a batch msg. + for i in range(5): + producer.send_async(b"hello-%d" % i, callback=None) + producer.flush() + + # Receive msgs and just ack 0, 1 msgs + results = [] + for i in range(5): + msg = consumer.receive() + print("receive from {}".format(msg.message_id())) + results.append(msg) + assert len(results) == 5 + for i in range(2): + consumer.acknowledge(results[i]) + time.sleep(0.2) + + # Restart consumer after, just receive 2~5 msg. + consumer.close() + consumer = client.subscribe(topic_name, + subscription_name='test-batch-index-ack', + batch_index_ack_enabled=True) + results2 = [] + for i in range(2, 5): + msg = consumer.receive() + results2.append(msg) + assert len(results2) == 3 + # assert no more msgs. + with self.assertRaises(pulsar.Timeout): + consumer.receive(timeout_millis=1000) + client.close() + + def test_dead_letter_policy_config(self): + with self.assertRaises(ValueError): + ConsumerDeadLetterPolicy(-1) + + policy = ConsumerDeadLetterPolicy(10) + self.assertEqual(10, policy.max_redeliver_count) + self.assertEqual("", policy.dead_letter_topic) + self.assertEqual("", policy.initial_subscription_name) + + def test_dead_letter_policy(self): + client = Client(self.serviceUrl) + topic = "my-python-topic-test-dlq" + str(time.time()) + dlq_topic = 'dlq-' + topic + max_redeliver_count = 5 + consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared, + dead_letter_policy=ConsumerDeadLetterPolicy(max_redeliver_count, dlq_topic, 'init-sub')) + dlq_consumer = client.subscribe(dlq_topic, "my-sub", consumer_type=ConsumerType.Shared) + + # Sen num msgs. + producer = client.create_producer(topic) + num = 10 + for i in range(num): + producer.send(b"hello-%d" % i) + producer.flush() + + # Redelivery all messages maxRedeliverCountNum time. + for i in range(1, num * max_redeliver_count + num + 1): + msg = consumer.receive() + if i % num == 0: + consumer.redeliver_unacknowledged_messages() + print(f"Start redeliver msgs '{i}'") + with self.assertRaises(pulsar.Timeout): + consumer.receive(100) + + for i in range(num): + msg = dlq_consumer.receive() + self.assertTrue(msg) + self.assertEqual(msg.data(), b"hello-%d" % i) + dlq_consumer.acknowledge(msg) + with self.assertRaises(pulsar.Timeout): + dlq_consumer.receive(100) + + client.close() + + def test_regex_subscription(self): + client = Client(self.serviceUrl) + topic1 = "persistent://public/default/test-regex-sub-1" + topic2 = "persistent://public/default/test-regex-sub-2" + topic3 = "non-persistent://public/default/test-regex-sub-3" + topic4 = "persistent://public/default/no-match-test-regex-sub-3" # no match pattern rule topic. + + producer1 = client.create_producer(topic1) + producer2 = client.create_producer(topic2) + producer3 = client.create_producer(topic3) + producer4 = client.create_producer(topic4) + + consumer_all = client.subscribe( + re.compile('public/default/test-regex-sub-.*'), "regex-sub-all", + consumer_type=ConsumerType.Shared, regex_subscription_mode=RegexSubscriptionMode.AllTopics + ) + + consumer_persistent = client.subscribe( + re.compile('public/default/test-regex-sub-.*'), "regex-sub-persistent", + consumer_type=ConsumerType.Shared, regex_subscription_mode=RegexSubscriptionMode.PersistentOnly + ) + + consumer_non_persistent = client.subscribe( + re.compile('public/default/test-regex-sub-.*'), "regex-sub-non-persistent", + consumer_type=ConsumerType.Shared, regex_subscription_mode=RegexSubscriptionMode.NonPersistentOnly + ) + + num = 10 + for i in range(num): + producer1.send(b"hello-1-%d" % i) + producer2.send(b"hello-2-%d" % i) + producer3.send(b"hello-3-%d" % i) + producer4.send(b"hello-4-%d" % i) + + # Assert consumer_all. + received_topics = set() + for i in range(3 * num): + msg = consumer_all.receive(TM) + topic_name = msg.topic_name() + self.assertIn(topic_name, [topic1, topic2, topic3]) + received_topics.add(topic_name) + consumer_all.acknowledge(msg) + self.assertEqual(received_topics, {topic1, topic2, topic3}) + with self.assertRaises(pulsar.Timeout): + consumer_all.receive(100) + + # Assert consumer_persistent. + received_topics.clear() + for i in range(2 * num): + msg = consumer_persistent.receive(TM) + topic_name = msg.topic_name() + self.assertIn(topic_name, [topic1, topic2]) + received_topics.add(topic_name) + consumer_persistent.acknowledge(msg) + self.assertEqual(received_topics, {topic1, topic2}) + with self.assertRaises(pulsar.Timeout): + consumer_persistent.receive(100) + + # Assert consumer_non_persistent. + received_topics.clear() + for i in range(num): + msg = consumer_non_persistent.receive(TM) + topic_name = msg.topic_name() + self.assertIn(topic_name, [topic3]) + received_topics.add(topic_name) + consumer_non_persistent.acknowledge(msg) + self.assertEqual(received_topics, {topic3}) + with self.assertRaises(pulsar.Timeout): + consumer_non_persistent.receive(100) + + client.close() if __name__ == "__main__": main() diff --git a/tests/test-conf/standalone-ssl.conf b/tests/test-conf/standalone-ssl.conf index 2ee4432..beed278 100644 --- a/tests/test-conf/standalone-ssl.conf +++ b/tests/test-conf/standalone-ssl.conf @@ -113,6 +113,9 @@ superUserRoles=localhost,superUser,admin brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= +# Enable batch index ACK +acknowledgmentAtBatchIndexLevelEnabled=true + ### --- BookKeeper Client --- ### # Authentication plugin to use when connecting to bookies diff --git a/tests/test-conf/standalone.conf b/tests/test-conf/standalone.conf index faa1277..0225e0d 100644 --- a/tests/test-conf/standalone.conf +++ b/tests/test-conf/standalone.conf @@ -100,6 +100,9 @@ superUserRoles= brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= +# Enable batch index ACK +acknowledgmentAtBatchIndexLevelEnabled=true + ### --- BookKeeper Client --- ###