diff --git a/CHANGELOG.md b/CHANGELOG.md index 23e8fa24..3d4af9a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.2.0](https://github.com/googleapis/java-pubsub-group-kafka-connector/compare/v1.1.0...v1.2.0) (2023-05-10) + + +### Features + +* Add the ability to set gRPC compression in the CPS sink connector. ([#259](https://github.com/googleapis/java-pubsub-group-kafka-connector/issues/259)) ([ad1e72b](https://github.com/googleapis/java-pubsub-group-kafka-connector/commit/ad1e72b7a09d6034c5ec0009131f39aaef236a50)) + ## [1.1.0](https://github.com/googleapis/java-pubsub-group-kafka-connector/compare/v1.0.0...v1.1.0) (2023-03-31) diff --git a/README.md b/README.md index 9dbdf075..b86b9353 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,8 @@ configurations: | headers.publish | Boolean | false | When true, include any headers as attributes when a message is published to Pub/Sub. | | orderingKeySource | String (none, key, partition) | none | When set to "none", do not set the ordering key. When set to "key", uses a message's key as the ordering key. If set to "partition", converts the partition number to a String and uses that as the ordering key. Note that using "partition" should only be used for low-throughput topics or topics with thousands of partitions. | | messageBodyName | String | "cps_message_body" | When using a struct or map value schema, this field or key name indicates that the corresponding value will go into the Pub/Sub message body. | +| enableCompression | Boolean | false | When true, enable [publish-side compression](https://cloud.google.com/pubsub/docs/publisher#compressing) in order to save on networking costs between Kafka Connect and Cloud Pub/Sub. | +| compressionBytesThreshold | Long | 240 | When enableCompression is true, the minimum size of publish request (in bytes) to compress. ### Pub/Sub Lite connector configs diff --git a/pom.xml b/pom.xml index d17aa38f..b6aacced 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 pubsub-group-kafka-connector - 1.1.0 + 1.2.0 jar Pub/Sub Group Kafka Connector https://github.com/googleapis/java-pubsub-group-kafka-connector diff --git a/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkConnector.java b/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkConnector.java index eb7d681f..e7c56118 100644 --- a/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkConnector.java +++ b/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkConnector.java @@ -47,6 +47,8 @@ public class CloudPubSubSinkConnector extends SinkConnector { public static final String MAX_REQUEST_TIMEOUT_MS = "maxRequestTimeoutMs"; public static final String MAX_TOTAL_TIMEOUT_MS = "maxTotalTimeoutMs"; public static final String MAX_SHUTDOWN_TIMEOUT_MS = "maxShutdownTimeoutMs"; + public static final String ENABLE_COMPRESSION = "enableCompression"; + public static final String COMPRESSION_BYTES_THRESHOLD = "compressionBytesThreshold"; public static final int DEFAULT_MAX_BUFFER_SIZE = 100; public static final long DEFAULT_MAX_BUFFER_BYTES = 9500000L; public static final int DEFAULT_DELAY_THRESHOLD_MS = 100; @@ -61,6 +63,8 @@ public class CloudPubSubSinkConnector extends SinkConnector { public static final String PUBLISH_KAFKA_HEADERS = "headers.publish"; public static final String ORDERING_KEY_SOURCE = "orderingKeySource"; public static final String DEFAULT_ORDERING_KEY_SOURCE = "none"; + public static final boolean DEFAULT_ENABLE_COMPRESSION = false; + public static final long DEFAULT_COMPRESSION_BYTES_THRESHOLD = 240L; /** Defines the accepted values for the {@link #ORDERING_KEY_SOURCE}. */ public enum OrderingKeySource { @@ -250,6 +254,20 @@ public ConfigDef config() { Importance.MEDIUM, "What to use to populate the Pub/Sub message ordering key. Possible values are " + "\"none\", \"key\", or \"partition\".") + .define( + ENABLE_COMPRESSION, + Type.BOOLEAN, + DEFAULT_ENABLE_COMPRESSION, + Importance.MEDIUM, + "When \"true\", use gRPC Gzip compression on publish requests before sending them " + + "to Cloud Pub/Sub.") + .define( + COMPRESSION_BYTES_THRESHOLD, + Type.LONG, + DEFAULT_COMPRESSION_BYTES_THRESHOLD, + Importance.MEDIUM, + "The number of bytes at which to compress a request when publishing to " + + "Cloud Pub/Sub. Only takes effect if \"enableCompression\" is \"true\".") .define( ConnectorUtils.CPS_ENDPOINT, Type.STRING, diff --git a/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java b/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java index 9faa7910..28c74ce0 100644 --- a/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java +++ b/src/main/java/com/google/pubsub/kafka/sink/CloudPubSubSinkTask.java @@ -81,6 +81,8 @@ public class CloudPubSubSinkTask extends SinkTask { private boolean includeMetadata; private boolean includeHeaders; private OrderingKeySource orderingKeySource; + private boolean enableCompression; + private long compressionBytesThreshold; private ConnectorCredentialsProvider gcpCredentialsProvider; private com.google.cloud.pubsub.v1.Publisher publisher; @@ -135,6 +137,9 @@ public void start(Map props) { orderingKeySource = OrderingKeySource.getEnum( (String) validatedProps.get(CloudPubSubSinkConnector.ORDERING_KEY_SOURCE)); + enableCompression = (Boolean) validatedProps.get(CloudPubSubSinkConnector.ENABLE_COMPRESSION); + compressionBytesThreshold = + (Long) validatedProps.get(CloudPubSubSinkConnector.COMPRESSION_BYTES_THRESHOLD); gcpCredentialsProvider = ConnectorCredentialsProvider.fromConfig(validatedProps); if (publisher == null) { // Only do this if we did not use the constructor. @@ -413,6 +418,10 @@ private void createPublisher() { if (orderingKeySource != OrderingKeySource.NONE) { builder.setEnableMessageOrdering(true); } + if (enableCompression) { + builder.setEnableCompression(true); + builder.setCompressionBytesThreshold(compressionBytesThreshold); + } try { publisher = builder.build(); } catch (Exception e) { diff --git a/versions.txt b/versions.txt index af274e56..2b77f448 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -pubsub-group-kafka-connector:1.1.0:1.1.0 +pubsub-group-kafka-connector:1.2.0:1.2.0